Initialize module and dependencies

vendor/golang.org/x/vuln/internal/client/client.go

@@ -0,0 +1,347 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package client provides an interface for accessing vulnerability
+// databases, via either HTTP or local filesystem access.
+//
+// The protocol is described at https://go.dev/security/vuln/database.
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/vuln/internal/derrors"
+	"golang.org/x/vuln/internal/osv"
+	isem "golang.org/x/vuln/internal/semver"
+	"golang.org/x/vuln/internal/web"
+)
+
+// A Client for reading vulnerability databases.
+type Client struct {
+	source
+}
+
+type Options struct {
+	HTTPClient *http.Client
+}
+
+// NewClient returns a client that reads the vulnerability database
+// in source (an "http" or "file" prefixed URL).
+//
+// It supports databases following the API described
+// in https://go.dev/security/vuln/database#api.
+func NewClient(source string, opts *Options) (_ *Client, err error) {
+	source = strings.TrimRight(source, "/")
+	uri, err := url.Parse(source)
+	if err != nil {
+		return nil, err
+	}
+	switch uri.Scheme {
+	case "http", "https":
+		return newHTTPClient(uri, opts)
+	case "file":
+		return newLocalClient(uri)
+	default:
+		return nil, fmt.Errorf("source %q has unsupported scheme", uri)
+	}
+}
+
+var errUnknownSchema = errors.New("unrecognized vulndb format; see https://go.dev/security/vuln/database#api for accepted schema")
+
+func newHTTPClient(uri *url.URL, opts *Options) (*Client, error) {
+	source := uri.String()
+
+	// v1 returns true if the source likely follows the V1 schema.
+	v1 := func() bool {
+		return source == "https://vuln.go.dev" ||
+			endpointExistsHTTP(source, "index/modules.json.gz")
+	}
+
+	if v1() {
+		return &Client{source: newHTTPSource(uri.String(), opts)}, nil
+	}
+
+	return nil, errUnknownSchema
+}
+
+func endpointExistsHTTP(source, endpoint string) bool {
+	r, err := http.Head(source + "/" + endpoint)
+	return err == nil && r.StatusCode == http.StatusOK
+}
+
+func newLocalClient(uri *url.URL) (*Client, error) {
+	dir, err := toDir(uri)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if the DB likely follows the v1 schema by
+	// looking for the "index/modules.json" endpoint.
+	if endpointExistsDir(dir, modulesEndpoint+".json") {
+		return &Client{source: newLocalSource(dir)}, nil
+	}
+
+	// If the DB doesn't follow the v1 schema,
+	// attempt to intepret it as a flat list of OSV files.
+	// This is currently a "hidden" feature, so don't output the
+	// specific error if this fails.
+	src, err := newHybridSource(dir)
+	if err != nil {
+		return nil, errUnknownSchema
+	}
+	return &Client{source: src}, nil
+}
+
+func toDir(uri *url.URL) (string, error) {
+	dir, err := web.URLToFilePath(uri)
+	if err != nil {
+		return "", err
+	}
+	fi, err := os.Stat(dir)
+	if err != nil {
+		return "", err
+	}
+	if !fi.IsDir() {
+		return "", fmt.Errorf("%s is not a directory", dir)
+	}
+	return dir, nil
+}
+
+func endpointExistsDir(dir, endpoint string) bool {
+	_, err := os.Stat(filepath.Join(dir, endpoint))
+	return err == nil
+}
+
+func NewInMemoryClient(entries []*osv.Entry) (*Client, error) {
+	s, err := newInMemorySource(entries)
+	if err != nil {
+		return nil, err
+	}
+	return &Client{source: s}, nil
+}
+
+func (c *Client) LastModifiedTime(ctx context.Context) (_ time.Time, err error) {
+	derrors.Wrap(&err, "LastModifiedTime()")
+
+	b, err := c.source.get(ctx, dbEndpoint)
+	if err != nil {
+		return time.Time{}, err
+	}
+
+	var dbMeta dbMeta
+	if err := json.Unmarshal(b, &dbMeta); err != nil {
+		return time.Time{}, err
+	}
+
+	return dbMeta.Modified, nil
+}
+
+type ModuleRequest struct {
+	// The module path to filter on.
+	// This must be set (if empty, ByModule errors).
+	Path string
+	// (Optional) If set, only return vulnerabilities affected
+	// at this version.
+	Version string
+}
+
+type ModuleResponse struct {
+	Path    string
+	Version string
+	Entries []*osv.Entry
+}
+
+// ByModules returns a list of responses
+// containing the OSV entries corresponding to each request.
+//
+// The order of the requests is preserved, and each request has
+// a response even if there are no entries (in which case the Entries
+// field is nil).
+func (c *Client) ByModules(ctx context.Context, reqs []*ModuleRequest) (_ []*ModuleResponse, err error) {
+	derrors.Wrap(&err, "ByModules(%v)", reqs)
+
+	metas, err := c.moduleMetas(ctx, reqs)
+	if err != nil {
+		return nil, err
+	}
+
+	resps := make([]*ModuleResponse, len(reqs))
+	g, gctx := errgroup.WithContext(ctx)
+	g.SetLimit(10)
+	for i, req := range reqs {
+		i, req := i, req
+		g.Go(func() error {
+			entries, err := c.byModule(gctx, req, metas[i])
+			if err != nil {
+				return err
+			}
+			resps[i] = &ModuleResponse{
+				Path:    req.Path,
+				Version: req.Version,
+				Entries: entries,
+			}
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+
+	return resps, nil
+}
+
+func (c *Client) moduleMetas(ctx context.Context, reqs []*ModuleRequest) (_ []*moduleMeta, err error) {
+	b, err := c.source.get(ctx, modulesEndpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	dec, err := newStreamDecoder(b)
+	if err != nil {
+		return nil, err
+	}
+
+	metas := make([]*moduleMeta, len(reqs))
+	for dec.More() {
+		var m moduleMeta
+		err := dec.Decode(&m)
+		if err != nil {
+			return nil, err
+		}
+		for i, req := range reqs {
+			if m.Path == req.Path {
+				metas[i] = &m
+			}
+		}
+	}
+
+	return metas, nil
+}
+
+// byModule returns the OSV entries matching the ModuleRequest,
+// or (nil, nil) if there are none.
+func (c *Client) byModule(ctx context.Context, req *ModuleRequest, m *moduleMeta) (_ []*osv.Entry, err error) {
+	// This module isn't in the database.
+	if m == nil {
+		return nil, nil
+	}
+
+	if req.Path == "" {
+		return nil, fmt.Errorf("module path must be set")
+	}
+
+	if req.Version != "" && !isem.Valid(req.Version) {
+		return nil, fmt.Errorf("version %s is not valid semver", req.Version)
+	}
+
+	var ids []string
+	for _, v := range m.Vulns {
+		if v.Fixed == "" || isem.Less(req.Version, v.Fixed) {
+			ids = append(ids, v.ID)
+		}
+	}
+
+	if len(ids) == 0 {
+		return nil, nil
+	}
+
+	entries, err := c.byIDs(ctx, ids)
+	if err != nil {
+		return nil, err
+	}
+
+	// Filter by version.
+	if req.Version != "" {
+		affected := func(e *osv.Entry) bool {
+			for _, a := range e.Affected {
+				if a.Module.Path == req.Path && isem.Affects(a.Ranges, req.Version) {
+					return true
+				}
+			}
+			return false
+		}
+
+		var filtered []*osv.Entry
+		for _, entry := range entries {
+			if affected(entry) {
+				filtered = append(filtered, entry)
+			}
+		}
+		if len(filtered) == 0 {
+			return nil, nil
+		}
+	}
+
+	sort.SliceStable(entries, func(i, j int) bool {
+		return entries[i].ID < entries[j].ID
+	})
+
+	return entries, nil
+}
+
+func (c *Client) byIDs(ctx context.Context, ids []string) (_ []*osv.Entry, err error) {
+	entries := make([]*osv.Entry, len(ids))
+	g, gctx := errgroup.WithContext(ctx)
+	g.SetLimit(10)
+	for i, id := range ids {
+		i, id := i, id
+		g.Go(func() error {
+			e, err := c.byID(gctx, id)
+			if err != nil {
+				return err
+			}
+			entries[i] = e
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+
+	return entries, nil
+}
+
+// byID returns the OSV entry with the given ID,
+// or an error if it does not exist / cannot be unmarshaled.
+func (c *Client) byID(ctx context.Context, id string) (_ *osv.Entry, err error) {
+	derrors.Wrap(&err, "byID(%s)", id)
+
+	b, err := c.source.get(ctx, entryEndpoint(id))
+	if err != nil {
+		return nil, err
+	}
+
+	var entry osv.Entry
+	if err := json.Unmarshal(b, &entry); err != nil {
+		return nil, err
+	}
+
+	return &entry, nil
+}
+
+// newStreamDecoder returns a decoder that can be used
+// to read an array of JSON objects.
+func newStreamDecoder(b []byte) (*json.Decoder, error) {
+	dec := json.NewDecoder(bytes.NewBuffer(b))
+
+	// skip open bracket
+	_, err := dec.Token()
+	if err != nil {
+		return nil, err
+	}
+
+	return dec, nil
+}

vendor/golang.org/x/vuln/internal/client/index.go

@@ -0,0 +1,120 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/vuln/internal/osv"
+	isem "golang.org/x/vuln/internal/semver"
+)
+
+// indexFromDir returns a raw index created from a directory
+// containing OSV entries.
+// It skips any non-JSON files but errors if any of the JSON files
+// cannot be unmarshaled into OSV, or have a filename other than <ID>.json.
+func indexFromDir(dir string) (map[string][]byte, error) {
+	idx := newIndex()
+	f := os.DirFS(dir)
+
+	if err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+		fname := d.Name()
+		ext := filepath.Ext(fname)
+		switch {
+		case err != nil:
+			return err
+		case d.IsDir():
+			return nil
+		case ext != ".json":
+			return nil
+		}
+
+		b, err := fs.ReadFile(f, d.Name())
+		if err != nil {
+			return err
+		}
+		var entry osv.Entry
+		if err := json.Unmarshal(b, &entry); err != nil {
+			return err
+		}
+		if fname != entry.ID+".json" {
+			return fmt.Errorf("OSV entries must have filename of the form <ID>.json, got %s", fname)
+		}
+
+		idx.add(&entry)
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return idx.raw()
+}
+
+func indexFromEntries(entries []*osv.Entry) (map[string][]byte, error) {
+	idx := newIndex()
+
+	for _, entry := range entries {
+		idx.add(entry)
+	}
+
+	return idx.raw()
+}
+
+type index struct {
+	db      *dbMeta
+	modules modulesIndex
+}
+
+func newIndex() *index {
+	return &index{
+		db:      &dbMeta{},
+		modules: make(map[string]*moduleMeta),
+	}
+}
+
+func (i *index) add(entry *osv.Entry) {
+	// Add to db index.
+	if entry.Modified.After(i.db.Modified) {
+		i.db.Modified = entry.Modified
+	}
+	// Add to modules index.
+	for _, affected := range entry.Affected {
+		modulePath := affected.Module.Path
+		if _, ok := i.modules[modulePath]; !ok {
+			i.modules[modulePath] = &moduleMeta{
+				Path:  modulePath,
+				Vulns: []moduleVuln{},
+			}
+		}
+		module := i.modules[modulePath]
+		module.Vulns = append(module.Vulns, moduleVuln{
+			ID:       entry.ID,
+			Modified: entry.Modified,
+			Fixed:    isem.NonSupersededFix(affected.Ranges),
+		})
+	}
+}
+
+func (i *index) raw() (map[string][]byte, error) {
+	data := make(map[string][]byte)
+
+	b, err := json.Marshal(i.db)
+	if err != nil {
+		return nil, err
+	}
+	data[dbEndpoint] = b
+
+	b, err = json.Marshal(i.modules)
+	if err != nil {
+		return nil, err
+	}
+	data[modulesEndpoint] = b
+
+	return data, nil
+}

vendor/golang.org/x/vuln/internal/client/schema.go

@@ -0,0 +1,77 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package client
+
+import (
+	"encoding/json"
+	"path"
+	"sort"
+	"time"
+)
+
+const (
+	idDir    = "ID"
+	indexDir = "index"
+)
+
+var (
+	dbEndpoint      = path.Join(indexDir, "db")
+	modulesEndpoint = path.Join(indexDir, "modules")
+)
+
+func entryEndpoint(id string) string {
+	return path.Join(idDir, id)
+}
+
+// dbMeta contains metadata about the database itself.
+type dbMeta struct {
+	// Modified is the time the database was last modified, calculated
+	// as the most recent time any single OSV entry was modified.
+	Modified time.Time `json:"modified"`
+}
+
+// moduleMeta contains metadata about a Go module that has one
+// or more vulnerabilities in the database.
+//
+// Found in the "index/modules" endpoint of the vulnerability database.
+type moduleMeta struct {
+	// Path is the module path.
+	Path string `json:"path"`
+	// Vulns is a list of vulnerabilities that affect this module.
+	Vulns []moduleVuln `json:"vulns"`
+}
+
+// moduleVuln contains metadata about a vulnerability that affects
+// a certain module.
+type moduleVuln struct {
+	// ID is a unique identifier for the vulnerability.
+	// The Go vulnerability database issues IDs of the form
+	// GO-<YEAR>-<ENTRYID>.
+	ID string `json:"id"`
+	// Modified is the time the vuln was last modified.
+	Modified time.Time `json:"modified"`
+	// Fixed is the latest version that introduces a fix for the
+	// vulnerability, in SemVer 2.0.0 format, with no leading "v" prefix.
+	Fixed string `json:"fixed,omitempty"`
+}
+
+// modulesIndex represents an in-memory modules index.
+type modulesIndex map[string]*moduleMeta
+
+func (m modulesIndex) MarshalJSON() ([]byte, error) {
+	modules := make([]*moduleMeta, 0, len(m))
+	for _, module := range m {
+		modules = append(modules, module)
+	}
+	sort.SliceStable(modules, func(i, j int) bool {
+		return modules[i].Path < modules[j].Path
+	})
+	for _, module := range modules {
+		sort.SliceStable(module.Vulns, func(i, j int) bool {
+			return module.Vulns[i].ID < module.Vulns[j].ID
+		})
+	}
+	return json.Marshal(modules)
+}

vendor/golang.org/x/vuln/internal/client/source.go

@@ -0,0 +1,150 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package client
+
+import (
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/vuln/internal/derrors"
+	"golang.org/x/vuln/internal/osv"
+)
+
+type source interface {
+	// get returns the raw, uncompressed bytes at the
+	// requested endpoint, which should be bare with no file extensions
+	// (e.g., "index/modules" instead of "index/modules.json.gz").
+	// It errors if the endpoint cannot be reached or does not exist
+	// in the expected form.
+	get(ctx context.Context, endpoint string) ([]byte, error)
+}
+
+func newHTTPSource(url string, opts *Options) *httpSource {
+	c := http.DefaultClient
+	if opts != nil && opts.HTTPClient != nil {
+		c = opts.HTTPClient
+	}
+	return &httpSource{url: url, c: c}
+}
+
+// httpSource reads a vulnerability database from an http(s) source.
+type httpSource struct {
+	url string
+	c   *http.Client
+}
+
+func (hs *httpSource) get(ctx context.Context, endpoint string) (_ []byte, err error) {
+	derrors.Wrap(&err, "get(%s)", endpoint)
+
+	method := http.MethodGet
+	reqURL := fmt.Sprintf("%s/%s", hs.url, endpoint+".json.gz")
+	req, err := http.NewRequestWithContext(ctx, method, reqURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := hs.c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("HTTP %s %s returned unexpected status: %s", method, reqURL, resp.Status)
+	}
+
+	// Uncompress the result.
+	r, err := gzip.NewReader(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	return io.ReadAll(r)
+}
+
+func newLocalSource(dir string) *localSource {
+	return &localSource{fs: os.DirFS(dir)}
+}
+
+// localSource reads a vulnerability database from a local file system.
+type localSource struct {
+	fs fs.FS
+}
+
+func (ls *localSource) get(ctx context.Context, endpoint string) (_ []byte, err error) {
+	derrors.Wrap(&err, "get(%s)", endpoint)
+
+	return fs.ReadFile(ls.fs, endpoint+".json")
+}
+
+func newHybridSource(dir string) (*hybridSource, error) {
+	index, err := indexFromDir(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	return &hybridSource{
+		index: &inMemorySource{data: index},
+		osv:   &localSource{fs: os.DirFS(dir)},
+	}, nil
+}
+
+// hybridSource reads OSV entries from a local file system, but reads
+// indexes from an in-memory map.
+type hybridSource struct {
+	index *inMemorySource
+	osv   *localSource
+}
+
+func (hs *hybridSource) get(ctx context.Context, endpoint string) (_ []byte, err error) {
+	derrors.Wrap(&err, "get(%s)", endpoint)
+
+	dir, file := filepath.Split(endpoint)
+
+	if filepath.Dir(dir) == indexDir {
+		return hs.index.get(ctx, endpoint)
+	}
+
+	return hs.osv.get(ctx, file)
+}
+
+// newInMemorySource creates a new in-memory source from OSV entries.
+// Adapted from x/vulndb/internal/database.go.
+func newInMemorySource(entries []*osv.Entry) (*inMemorySource, error) {
+	data, err := indexFromEntries(entries)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, entry := range entries {
+		b, err := json.Marshal(entry)
+		if err != nil {
+			return nil, err
+		}
+		data[entryEndpoint(entry.ID)] = b
+	}
+
+	return &inMemorySource{data: data}, nil
+}
+
+// inMemorySource reads databases from an in-memory map.
+// Currently intended for use only in unit tests.
+type inMemorySource struct {
+	data map[string][]byte
+}
+
+func (db *inMemorySource) get(ctx context.Context, endpoint string) ([]byte, error) {
+	b, ok := db.data[endpoint]
+	if !ok {
+		return nil, fmt.Errorf("no data found at endpoint %q", endpoint)
+	}
+	return b, nil
+}