Initial Commit

This commit is contained in:
2022-11-02 22:29:44 -07:00
commit 1fa1d63307
14 changed files with 1626 additions and 0 deletions

384
internal/analyze/analyze.go Normal file
View File

@@ -0,0 +1,384 @@
package analyze
import (
"encoding/hex"
"net/mail"
"net/url"
"strings"
"go.arsenm.dev/lure-repo-bot/internal/spdx"
"golang.org/x/exp/slices"
"mvdan.cc/sh/v3/expand"
"mvdan.cc/sh/v3/interp"
"mvdan.cc/sh/v3/syntax"
)
type Finding struct {
ItemType string
ItemName string
Line uint
Index any
Msg string
ExtraMsg string
}
func AnalyzeScript(r *interp.Runner, fl *syntax.File) ([]Finding, error) {
var findings []Finding
if _, ok := r.Vars["name"]; !ok {
findings = append(findings, Finding{
ItemType: "variable",
ItemName: "name",
Msg: "The %s is required",
})
}
if _, ok := r.Vars["version"]; !ok {
findings = append(findings, Finding{
ItemType: "variable",
ItemName: "version",
Msg: "The %s is required",
})
}
if _, ok := r.Vars["release"]; !ok {
findings = append(findings, Finding{
ItemType: "variable",
ItemName: "release",
Msg: "The %s is required",
})
}
if _, ok := r.Funcs["package"]; !ok {
findings = append(findings, Finding{
ItemType: "function",
ItemName: "package",
Msg: "The %s is required",
})
}
for name, scriptVar := range r.Vars {
_, scriptVar = scriptVar.Resolve(r.Env)
val := getVal(&scriptVar)
// Remove any override suffix, so that we
// check all the overrides as well
cutName, _, _ := strings.Cut(name, "_")
// build_vars has an underscore, and thus is a special case
// that must be accounted for
if strings.HasPrefix("name", "build_vars") {
cutName = "build_vars"
}
switch cutName {
case "release":
valStr, ok := mustBeStr(val, name, &findings)
if !ok {
continue
}
if !isNumeric(strings.TrimPrefix(valStr, "-")) {
findings = append(findings, Finding{
ItemType: "variable",
ItemName: name,
Msg: "The %s must be an integer",
})
continue
}
case "epoch":
valStr, ok := mustBeStr(val, name, &findings)
if !ok {
continue
}
if !isNumeric(valStr) {
findings = append(findings, Finding{
ItemType: "variable",
ItemName: name,
Msg: "The %s must be a positive integer",
})
continue
}
case "homepage":
valStr, ok := mustBeStr(val, name, &findings)
if !ok {
continue
}
_, err := url.ParseRequestURI(valStr)
if err != nil {
findings = append(findings, Finding{
ItemType: "variable",
ItemName: name,
Msg: "The %s must be a valid URL",
})
continue
}
case "maintainer":
valStr, ok := mustBeStr(val, name, &findings)
if !ok {
continue
}
addr, err := mail.ParseAddress(valStr)
if err != nil {
findings = append(findings, Finding{
ItemType: "variable",
ItemName: name,
Msg: "The %s must be a valid RFC 5322 address",
})
continue
}
if addr.Name == "" || addr.Address == "" {
findings = append(findings, Finding{
ItemType: "variable",
ItemName: name,
Msg: "The %s must contain a name and email (e.g. Arsen Musayelyan <arsen@arsenm.dev>)",
})
continue
}
case "architectures":
valSlice, ok := mustBeArray(val, name, &findings)
if !ok {
continue
}
if slices.Contains(valSlice, "noarch") || slices.Contains(valSlice, "any") {
findings = append(findings, Finding{
ItemType: "variable",
ItemName: name,
Msg: "The %s must be set to 'all' to represent noarch/any",
})
continue
}
case "licenses":
valSlice, ok := mustBeArray(val, name, &findings)
if !ok {
continue
}
for _, val := range valSlice {
if strings.Contains(strings.ToLower(val), "custom") {
continue
}
license := spdx.Licenses.License(val)
if license == nil {
similar := spdx.FindSimilarLicense(val)
f := Finding{
ItemType: "variable",
ItemName: name,
Msg: "The %s contains an invalid SPDX license identifier: '" + val + "'.",
ExtraMsg: "A list of SPDX license identifiers can be found at https://spdx.org/licenses/.",
}
if similar != "" {
f.Msg += " Did you mean '" + similar + "'?"
}
findings = append(findings, f)
continue
}
}
case "provides":
mustBeArray(val, name, &findings)
case "conflicts":
mustBeArray(val, name, &findings)
case "deps":
mustBeArray(val, name, &findings)
case "build_deps":
mustBeArray(val, name, &findings)
case "replaces":
mustBeArray(val, name, &findings)
case "sources":
valSlice, ok := mustBeArray(val, name, &findings)
if !ok {
continue
}
for i, val := range valSlice {
u, err := url.ParseRequestURI(val)
if err != nil {
findings = append(findings, Finding{
ItemType: "element",
Index: i,
ItemName: name,
Msg: "The %s must be a valid URL",
})
continue
}
query := u.Query()
var validParams []string
if strings.HasPrefix(u.Scheme, "git+") {
validParams = []string{"tag", "branch", "commit", "depth", "name"}
} else {
validParams = []string{"archive"}
}
for paramName := range query {
if strings.HasPrefix(paramName, "~") {
paramName = strings.TrimPrefix(paramName, "~")
} else {
continue
}
if !slices.Contains(validParams, paramName) {
findings = append(findings, Finding{
ItemType: "element",
ItemName: name,
Index: i,
Msg: "The %s contains an invalid parameter name '~" + paramName + "'",
})
continue
}
}
}
case "checksums":
valSlice, ok := mustBeArray(val, name, &findings)
if !ok {
continue
}
sourcesName := strings.Replace(name, "checksums", "sources", 1)
srcs, ok := r.Vars[sourcesName]
if !ok || len(srcs.List) != len(valSlice) {
findings = append(findings, Finding{
ItemType: "array",
ItemName: name,
Msg: "The %s is not the same size as its corresponding sources array",
})
}
for i, val := range valSlice {
if val != "SKIP" && len(val) != 64 {
findings = append(findings, Finding{
ItemType: "element",
ItemName: name,
Index: i,
Msg: "The %s contains an invalid SHA256 checksum. SHA256 hashes must be 64 characters in length.",
})
continue
}
if val != "SKIP" {
_, err := hex.DecodeString(val)
if err != nil {
findings = append(findings, Finding{
ItemType: "element",
ItemName: name,
Index: i,
Msg: "The %s contains an invalid SHA256 checksum. SHA256 hashes must be valid hexadecimal.",
})
continue
}
}
}
case "backup":
mustBeArray(val, name, &findings)
case "scripts":
mustBeMap(val, name, &findings)
}
}
lns := FindLines(fl)
for i, finding := range findings {
if finding.ItemType == "function" {
ln, ok := lns.Funcs[finding.ItemName]
if ok {
findings[i].Line = ln
}
} else {
ln, ok := lns.Vars[finding.ItemName]
if ok {
findings[i].Line = ln
}
}
}
return findings, nil
}
func mustBeStr(val any, name string, findings *[]Finding) (string, bool) {
valStr, ok := val.(string)
if !ok {
*findings = append(*findings, Finding{
ItemType: "variable",
ItemName: name,
Msg: "The %s must be a string",
})
}
return valStr, ok
}
func mustBeArray(val any, name string, findings *[]Finding) ([]string, bool) {
valSlice, ok := val.([]string)
if !ok {
*findings = append(*findings, Finding{
ItemType: "variable",
ItemName: name,
Msg: "The %s must be an array",
})
}
return valSlice, ok
}
func mustBeMap(val any, name string, findings *[]Finding) (map[string]string, bool) {
valMap, ok := val.(map[string]string)
if !ok {
*findings = append(*findings, Finding{
ItemType: "variable",
ItemName: name,
Msg: "The %s must be a map",
})
}
return valMap, ok
}
func getVal(v *expand.Variable) any {
if v.Str != "" {
return v.Str
} else if v.List != nil {
return v.List
} else if v.Map != nil {
return v.Map
}
return nil
}
func isNumeric(s string) bool {
index := strings.IndexFunc(s, func(r rune) bool {
return r < '0' || r > '9'
})
return index == -1
}
type Lines struct {
Vars map[string]uint
Funcs map[string]uint
}
func FindLines(fl *syntax.File) Lines {
out := Lines{map[string]uint{}, map[string]uint{}}
for _, stmt := range fl.Stmts {
switch cmd := stmt.Cmd.(type) {
case *syntax.CallExpr:
if len(cmd.Assigns) == 0 {
continue
}
name := cmd.Assigns[0].Name.Value
line := cmd.Assigns[0].Pos().Line()
out.Vars[name] = line
case *syntax.FuncDecl:
out.Funcs[cmd.Name.Value] = cmd.Pos().Line()
}
}
return out
}

47
internal/queue/queue.go Normal file
View File

@@ -0,0 +1,47 @@
// Package queue provides an unbounded FIFO queue
// for payloads received via webhooks.
package queue
import (
"container/list"
)
type Queue[T any] struct {
buf *list.List
out chan T
valAdded chan struct{}
}
func New[T any]() *Queue[T] {
q := &Queue[T]{
buf: list.New().Init(),
out: make(chan T),
valAdded: make(chan struct{}),
}
go func() {
for {
if q.buf.Len() == 0 {
<-q.valAdded
}
e := q.buf.Front()
q.out <- e.Value.(T)
q.buf.Remove(e)
}
}()
return q
}
func (q *Queue[T]) Add(val T) {
q.buf.PushBack(val)
select {
case q.valAdded <- struct{}{}:
default:
}
}
func (q *Queue[T]) Channel() <-chan T {
return q.out
}

55
internal/shutils/nop.go Normal file
View File

@@ -0,0 +1,55 @@
/*
* LURE - Linux User REpository
* Copyright (C) 2022 Arsen Musayelyan
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package shutils
import (
"context"
"io"
"os"
)
func NopReadDir(context.Context, string) ([]os.FileInfo, error) {
return nil, os.ErrNotExist
}
func NopStat(context.Context, string, bool) (os.FileInfo, error) {
return nil, os.ErrNotExist
}
func NopExec(context.Context, []string) error {
return nil
}
func NopOpen(context.Context, string, int, os.FileMode) (io.ReadWriteCloser, error) {
return NopRWC{}, nil
}
type NopRWC struct{}
func (NopRWC) Read([]byte) (int, error) {
return 0, io.EOF
}
func (NopRWC) Write([]byte) (int, error) {
return 0, io.EOF
}
func (NopRWC) Close() error {
return nil
}

102
internal/spdx/spdx.go Normal file
View File

@@ -0,0 +1,102 @@
package spdx
import (
"context"
"log"
"sync"
"time"
"github.com/adrg/strutil/metrics"
"github.com/mitchellh/go-spdx"
)
type syncLicenseList struct {
*spdx.LicenseList
*sync.Mutex
}
var Licenses = syncLicenseList{
LicenseList: &spdx.LicenseList{},
Mutex: &sync.Mutex{},
}
func (sll syncLicenseList) License(id string) *spdx.LicenseInfo {
sll.Lock()
l := sll.LicenseList.License(id)
sll.Unlock()
return l
}
func StartUpdater(ctx context.Context) {
err := Update()
if err != nil {
log.Fatalln("Error updating SPDX license list:", err)
}
ticker := time.NewTicker(time.Hour)
go func() {
for {
select {
case <-ticker.C:
err = Update()
if err != nil {
log.Println("Error updating SPDX license list:", err)
}
case <-ctx.Done():
ticker.Stop()
return
}
}
}()
}
func Update() error {
list, err := spdx.List()
if err != nil {
return err
}
Licenses.Lock()
Licenses.LicenseList = list
Licenses.Unlock()
return nil
}
// findSimilar finds the most similar license ID
// to the one provided
func FindSimilarLicense(s string) string {
Licenses.Lock()
defer Licenses.Unlock()
jw := metrics.NewJaroWinkler()
jw.CaseSensitive = false
sims := make([]float64, len(Licenses.Licenses))
for i, license := range Licenses.Licenses {
sims[i] = jw.Compare(s, license.ID)
}
index := maxIndex(sims)
if index == -1 {
return ""
} else {
return Licenses.Licenses[index].ID
}
}
func maxIndex(ff []float64) int {
if len(ff) == 0 {
return -1
}
m := ff[0]
mi := 0
for i, f := range ff {
if f > m {
m = f
mi = i
}
}
return mi
}

310
internal/types/github.go Normal file
View File

@@ -0,0 +1,310 @@
package types
import "time"
type PullRequestPayload struct {
IsGitea bool `json:"-"`
Action string `json:"action"`
Number int64 `json:"number"`
Changes struct {
Title struct {
From string `json:"from"`
} `json:"title"`
Body struct {
From string `json:"from"`
} `json:"body"`
} `json:"changes"`
Label Label `json:"label"`
PullRequest PullRequest `json:"pull_request"`
Repository Repository `json:"repository"`
Organization Organization `json:"organization"`
Sender User `json:"sender"`
}
type Repository struct {
ID int64 `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
Owner User `json:"owner"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
Fork bool `json:"fork"`
URL string `json:"url"`
ForksURL string `json:"forks_url"`
KeysURL string `json:"keys_url"`
CollaboratorsURL string `json:"collaborators_url"`
TeamsURL string `json:"teams_url"`
HooksURL string `json:"hooks_url"`
IssueEventsURL string `json:"issue_events_url"`
EventsURL string `json:"events_url"`
AssigneesURL string `json:"assignees_url"`
BranchesURL string `json:"branches_url"`
TagsURL string `json:"tags_url"`
BlobsURL string `json:"blobs_url"`
GitTagsURL string `json:"git_tags_url"`
GitRefsURL string `json:"git_refs_url"`
TreesURL string `json:"trees_url"`
StatusesURL string `json:"statuses_url"`
LanguagesURL string `json:"languages_url"`
StargazersURL string `json:"stargazers_url"`
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentURL string `json:"issue_comment_url"`
ContentsURL string `json:"contents_url"`
CompareURL string `json:"compare_url"`
MergesURL string `json:"merges_url"`
ArchiveURL string `json:"archive_url"`
DownloadsURL string `json:"downloads_url"`
IssuesURL string `json:"issues_url"`
PullsURL string `json:"pulls_url"`
MilestonesURL string `json:"milestones_url"`
NotificationsURL string `json:"notifications_url"`
LabelsURL string `json:"labels_url"`
ReleasesURL string `json:"releases_url"`
DeploymentsURL string `json:"deployments_url"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
PushedAt time.Time `json:"pushed_at"`
GitURL string `json:"git_url"`
SSHURL string `json:"ssh_url"`
CloneURL string `json:"clone_url"`
SvnURL string `json:"svn_url"`
Homepage string `json:"homepage"`
Size int64 `json:"size"`
StargazersCount int64 `json:"stargazers_count"`
WatchersCount int64 `json:"watchers_count"`
Language string `json:"language"`
HasIssues bool `json:"has_issues"`
HasProjects bool `json:"has_projects"`
HasDownloads bool `json:"has_downloads"`
HasWiki bool `json:"has_wiki"`
HasPages bool `json:"has_pages"`
ForksCount int64 `json:"forks_count"`
MirrorURL string `json:"mirror_url"`
Archived bool `json:"archived"`
Disabled bool `json:"disabled"`
OpenIssuesCount int64 `json:"open_issues_count"`
License License `json:"license"`
Forks int64 `json:"forks"`
OpenIssues int64 `json:"open_issues"`
Watchers int64 `json:"watchers"`
DefaultBranch string `json:"default_branch"`
}
type License struct {
Key string `json:"key"`
Name string `json:"name"`
URL string `json:"url"`
SpdxID string `json:"spdx_id"`
NodeID string `json:"node_id"`
HTMLURL string `json:"html_url"`
}
type PullRequest struct {
URL string `json:"url"`
ID int64 `json:"id"`
NodeID string `json:"node_id"`
HTMLURL string `json:"html_url"`
DiffURL string `json:"diff_url"`
PatchURL string `json:"patch_url"`
IssueURL string `json:"issue_url"`
CommitsURL string `json:"commits_url"`
ReviewCommentsURL string `json:"review_comments_url"`
ReviewCommentURL string `json:"review_comment_url"`
CommentsURL string `json:"comments_url"`
StatusesURL string `json:"statuses_url"`
Number int64 `json:"number"`
State string `json:"state"`
Locked bool `json:"locked"`
Title string `json:"title"`
User User `json:"user"`
Body string `json:"body"`
Labels []Label `json:"labels"`
Milestone Milestone `json:"milestone"`
ActiveLockReason string `json:"active_lock_reason"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ClosedAt time.Time `json:"closed_at"`
MergedAt time.Time `json:"merged_at"`
MergeCommitSha string `json:"merge_commit_sha"`
Assignee User `json:"assignee"`
Assignees []User `json:"assignees"`
RequestedReviewers []User `json:"requested_reviewers"`
RequestedTeams []Team `json:"requested_teams"`
Head Commit `json:"head"`
Base Commit `json:"base"`
Links struct {
Self struct {
Href string `json:"href"`
} `json:"self"`
HTML struct {
Href string `json:"href"`
} `json:"html"`
Issue struct {
Href string `json:"href"`
} `json:"issue"`
Comments struct {
Href string `json:"href"`
} `json:"comments"`
ReviewComments struct {
Href string `json:"href"`
} `json:"review_comments"`
ReviewComment struct {
Href string `json:"href"`
} `json:"review_comment"`
Commits struct {
Href string `json:"href"`
} `json:"commits"`
Statuses struct {
Href string `json:"href"`
} `json:"statuses"`
} `json:"_links"`
AuthorAssociation string `json:"author_association"`
Draft bool `json:"draft"`
Merged bool `json:"merged"`
Mergeable bool `json:"mergeable"`
Rebaseable bool `json:"rebaseable"`
MergeableState string `json:"mergeable_state"`
MergedBy User `json:"merged_by"`
Comments int64 `json:"comments"`
ReviewComments int64 `json:"review_comments"`
MaintainerCanModify bool `json:"maintainer_can_modify"`
Commits int64 `json:"commits"`
Additions int64 `json:"additions"`
Deletions int64 `json:"deletions"`
ChangedFiles int64 `json:"changed_files"`
}
type Label struct {
ID int64 `json:"id"`
NodeID string `json:"node_id"`
URL string `json:"url"`
Name string `json:"name"`
Color string `json:"color"`
Default bool `json:"default"`
Description string `json:"description"`
}
type Milestone struct {
URL string `json:"url"`
HTMLURL string `json:"html_url"`
LabelsURL string `json:"labels_url"`
ID int64 `json:"id"`
NodeID string `json:"node_id"`
Number int `json:"number"`
Title string `json:"title"`
Description string `json:"description"`
Creator User `json:"creator"`
OpenIssues int64 `json:"open_issues"`
ClosedIssues int64 `json:"closed_issues"`
State string `json:"state"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
DueOn *time.Time `json:"due_on"`
ClosedAt time.Time `json:"closed_at"`
}
type User struct {
Login string `json:"login"`
ID int64 `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
}
type Team struct {
ID int64 `json:"id"`
NodeID string `json:"node_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
Name string `json:"name"`
Slug string `json:"slug"`
Description string `json:"description"`
Privacy string `json:"privacy"`
Permission string `json:"permission"`
MembersURL string `json:"members_url"`
RepositoriesURL string `json:"repositories_url"`
}
type Commit struct {
Label string `json:"label"`
Ref string `json:"ref"`
Sha string `json:"sha"`
User User `json:"user"`
Repo Repository `json:"repo"`
}
type Organization struct {
Login string `json:"login"`
ID int64 `json:"id"`
NodeID string `json:"node_id"`
URL string `json:"url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
HooksURL string `json:"hooks_url"`
IssuesURL string `json:"issues_url"`
MembersURL string `json:"members_url"`
PublicMembersURL string `json:"public_members_url"`
AvatarURL string `json:"avatar_url"`
Description string `json:"description"`
Name string `json:"name"`
Company string `json:"company"`
Blog string `json:"blog"`
Location string `json:"location"`
Email string `json:"email"`
TwitterUsername string `json:"twitter_username"`
IsVerified bool `json:"is_verified"`
HasOrganizationProjects bool `json:"has_organization_projects"`
HasRepositoryProjects bool `json:"has_repository_projects"`
PublicRepos int64 `json:"public_repos"`
PublicGists int64 `json:"public_gists"`
Followers int64 `json:"followers"`
Following int64 `json:"following"`
HTMLURL string `json:"html_url"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Type string `json:"type"`
TotalPrivateRepos int64 `json:"total_private_repos"`
OwnedPrivateRepos int64 `json:"owned_private_repos"`
PrivateGists int64 `json:"private_gists"`
DiskUsage int64 `json:"disk_usage"`
Collaborators int64 `json:"collaborators"`
BillingEmail string `json:"billing_email"`
Plan Plan `json:"plan"`
DefaultRepositoryPermission string `json:"default_repository_permission"`
MembersCanCreateRepositories bool `json:"members_can_create_repositories"`
TwoFactorRequirementEnabled bool `json:"two_factor_requirement_enabled"`
MembersAllowedRepositoryCreationType string `json:"members_allowed_repository_creation_type"`
MembersCanCreatePublicRepositories bool `json:"members_can_create_public_repositories"`
MembersCanCreatePrivateRepositories bool `json:"members_can_create_private_repositories"`
MembersCanCreateint64ernalRepositories bool `json:"members_can_create_int64ernal_repositories"`
MembersCanCreatePages bool `json:"members_can_create_pages"`
MembersCanForkPrivateRepositories bool `json:"members_can_fork_private_repositories"`
}
type Plan struct {
Name string `json:"name"`
Space int64 `json:"space"`
PrivateRepos int64 `json:"private_repos"`
FilledSeats int64 `json:"filled_seats"`
Seats int64 `json:"seats"`
}