analysis

README

Configs and code to run analysis jobs

Cluster

To access the cluster, run:

gcloud container clusters get-credentials analysis-cluster --zone=us-central1-c --project=ossf-malware-analysis

Setup

Workload Identity is enabled for uploads to GCS.

Deployment

The code in this directory is by building the docker image in build/analysis.

Analysis

The two scripts in this repo can be run directly:

./python-runner.sh
./node-runner.sh

To clean up, run:

kubectl delete pod -l install=1

The .txt files contain package data from NPM and PyPI.

Local usage

To run the analysis code locally, the easiest way is to use the Docker image gcr.io/ossf-malware-analysis/analysis. This can be built from ../build/build_docker.sh.

This container uses podman to run a nested, sandboxed (gVisor) container for analysis.

The following commands will dump the JSON result to /tmp/results.

Live package

To run this on a live package (e.g. the "Django" package on https://pypi.org)

$ mkdir /tmp/results
$ docker run --privileged -ti \
    -v /tmp/results:/results \
    gcr.io/ossf-malware-analysis/analysis analyze \
    -package Django -ecosystem pypi \
    -upload file:///results/

Local package

To run this on a local package archive (e.g. /path/to/test.whl), it needs to be mounted into the the container.

$ mkdir /tmp/results
$ docker run --privileged -ti \
    -v /tmp/results:/results \
    -v /path/to/test.whl:/test.whl \
    gcr.io/ossf-malware-analysis/analysis analyze \
    -local /test.whl -ecosystem pypi \
    -upload file:///results/

Documentation

Index

• Variables
• func UploadResults(ctx context.Context, bucket, path string, result *AnalysisResult) error
• func WriteResultsToDocstore(ctx context.Context, collectionPrefix string, result *AnalysisResult) error
• type AnalysisResult
  • func RunLive(ecosystem, pkgName, version, image, command string) *AnalysisResult
  • func RunLocal(ecosystem, pkgPath, version, image, command string) *AnalysisResult
  • func (r *AnalysisResult) GenerateCmdIndexes() []*DocstoreIndex
  • func (r *AnalysisResult) GenerateFileIndexes() []*DocstoreIndex
  • func (r *AnalysisResult) GenerateSocketIndexes() []*DocstoreIndex
• type DocstoreIndex
• type Package
• type PkgManager

Variables

var NPMPackageManager = PkgManager{
	Image: "gcr.io/ossf-malware-analysis/node",
	CommandFmt: func(pkg, ver string) string {
		if ver != "" {
			return fmt.Sprintf("analyze.js %s@%s", pkg, ver)
		}

		return fmt.Sprintf("analyze.js %s", pkg)
	},
	GetLatest: getNPMLatest,
}

var PyPIPackageManager = PkgManager{
	Image: "gcr.io/ossf-malware-analysis/python",
	CommandFmt: func(pkg, ver string) string {
		if ver != "" {
			return fmt.Sprintf("analyze.py %s==%s", pkg, ver)
		}

		return fmt.Sprintf("analyze.py %s", pkg)
	},
	GetLatest: getPyPILatest,
}

var RubyGemsPackageManager = PkgManager{
	Image: "gcr.io/ossf-malware-analysis/ruby",
	CommandFmt: func(pkg, ver string) string {
		if ver != "" {
			return fmt.Sprintf("analyze.rb %s %s", pkg, ver)
		}
		return fmt.Sprintf("analyze.rb %s", pkg)
	},
	GetLatest: getRubyGemsLatest,
}

var (
	SupportedPkgManagers = map[string]PkgManager{
		"npm":      NPMPackageManager,
		"pypi":     PyPIPackageManager,
		"rubygems": RubyGemsPackageManager,
	}
)

Functions

func UploadResults

func UploadResults(ctx context.Context, bucket, path string, result *AnalysisResult) error

func WriteResultsToDocstore

func WriteResultsToDocstore(ctx context.Context, collectionPrefix string, result *AnalysisResult) error

Types

type AnalysisResult

type AnalysisResult struct {
	Package  Package
	Files    []fileResult
	Sockets  []socketInfo
	Commands []commandResult
}

func RunLive

func RunLive(ecosystem, pkgName, version, image, command string) *AnalysisResult

func RunLocal

func RunLocal(ecosystem, pkgPath, version, image, command string) *AnalysisResult

func (*AnalysisResult) GenerateCmdIndexes

func (r *AnalysisResult) GenerateCmdIndexes() []*DocstoreIndex

func (*AnalysisResult) GenerateFileIndexes

func (r *AnalysisResult) GenerateFileIndexes() []*DocstoreIndex

func (*AnalysisResult) GenerateSocketIndexes

func (r *AnalysisResult) GenerateSocketIndexes() []*DocstoreIndex

type DocstoreIndex

type DocstoreIndex struct {
	ID      string
	Package Package
	Indexes []string
}

type Package

type Package struct {
	Ecosystem string
	Name      string
	Version   string
}

type PkgManager

type PkgManager struct {
	CommandFmt func(string, string) string
	GetLatest  func(string) string
	Image      string
}