From 5aec0a0895b1277b8ca2499f78beac635ba4f5ce Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Thu, 3 Aug 2023 00:30:19 +0300 Subject: [PATCH] initial import Signed-off-by: Vasiliy Tolstov --- .gitignore | 1 + entities.go | 24 +++++ go.mod | 40 +++++++ go.sum | 156 ++++++++++++++++++++++++++++ main.go | 159 ++++++++++++++++++++++++++++ modproxy.go | 292 ++++++++++++++++++++++++++++++++++++++++++++++++++++ packages.go | 115 +++++++++++++++++++++ 7 files changed, 787 insertions(+) create mode 100644 entities.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 modproxy.go create mode 100644 packages.go diff --git a/.gitignore b/.gitignore index adf8f72..4b3bdc8 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ # Go workspace file go.work +/pkgdash diff --git a/entities.go b/entities.go new file mode 100644 index 0000000..6f53222 --- /dev/null +++ b/entities.go @@ -0,0 +1,24 @@ +package main + +type DBPackage struct { + Name string `db:"name"` // service name, last component path + URL string `db:"url"` // scm url + Modules []int64 `db:"modules"` // parsed go.mod modules + ID int64 `db:"id"` // package id + Issues []int64 `db:"issues"` // issues list +} + +type DBModule struct { + Name string `db:"name"` // module name + Version string `db:"version"` // module + ID int64 `db:"id"` + Package int64 `db:"package"` +} + +type DBIssue struct { + ID int64 `db:"id"` + Package int64 `db:"package"` + Modules []int64 `db:"modules"` + Status int64 `db:"status"` + Desc string `db:"desc"` +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..3529e79 --- /dev/null +++ b/go.mod @@ -0,0 +1,40 @@ +module go.unistack.org/unistack-org/pkgdash + +go 1.20 + +require ( + github.com/go-git/go-git/v5 v5.8.1 + go.unistack.org/micro-codec-yaml/v4 v4.0.0 + go.unistack.org/micro-config-flag/v4 v4.0.2 + go.unistack.org/micro/v4 v4.0.5 +) + +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect + github.com/acomagu/bufpipe v1.0.4 // indirect + github.com/cloudflare/circl v1.3.3 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.4.1 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/sergi/go-diff v1.3.1 // indirect + github.com/skeema/knownhosts v1.2.0 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + go.unistack.org/micro-proto/v4 v4.0.1 // indirect + golang.org/x/crypto v0.11.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/tools v0.11.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d351acd --- /dev/null +++ b/go.sum @@ -0,0 +1,156 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 h1:KLq8BE0KwCL+mmXnjLWEAOYO+2l2AE4YMmqG1ZpZHBs= +github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= +github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= +github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= +github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A= +github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM= +github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.unistack.org/micro-codec-yaml/v4 v4.0.0 h1:DK/+fA69gmnvYmGHe3kqVR0pukS8ank5lPG4MfdPQEk= +go.unistack.org/micro-codec-yaml/v4 v4.0.0/go.mod h1:UG4ouILbUfgNL7nvp7CpmD2IIzO8bQHjzDIahOvIAgM= +go.unistack.org/micro-config-flag/v4 v4.0.2 h1:qoDpT/H8a8TYZS9ucoNC0bLhNdp+UFlhnRWOuqIKAyo= +go.unistack.org/micro-config-flag/v4 v4.0.2/go.mod h1:s0AYvz8rRtMHG2tkXoSmF8xFuHG12x3v1NERdMHDdiQ= +go.unistack.org/micro-proto/v4 v4.0.1 h1:2RKHgtCOOcAFgKsnngGK5bqM/6MWXOjVCdw03dbuoF8= +go.unistack.org/micro-proto/v4 v4.0.1/go.mod h1:ArmK7o+uFvxSY3dbJhKBBX4Pm1rhWdLEFf3LxBrMtec= +go.unistack.org/micro/v4 v4.0.5 h1:uacmV8yHjCOtVfECc4w1tgp6bd3bBWrZa8X4iQLk48g= +go.unistack.org/micro/v4 v4.0.5/go.mod h1:bVEYTlPi0EsdgZZt311bIroDg9ict7ky3C87dSCCAGk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= +golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/main.go b/main.go new file mode 100644 index 0000000..6b316cd --- /dev/null +++ b/main.go @@ -0,0 +1,159 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "net/url" + "os" + "sort" + "strings" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/filemode" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage/memory" + "github.com/jmoiron/sqlx" + flagconfig "go.unistack.org/micro-config-flag/v4" + "go.unistack.org/micro/v4/config" + "go.unistack.org/micro/v4/logger" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" +) + +type Config struct { + Action string `flag:"name=action,desc='action to run',default='add'"` + Url string `flag:"name=url,desc='url or repo',default=''"` + CheckInterval string `default:"*/10 * * * *"` +} + +func main() { + var err error + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cfg := &Config{} + + if err = config.Load(ctx, []config.Config{ + config.NewConfig(config.Struct(cfg)), + flagconfig.NewConfig(config.Struct(cfg)), + }); err != nil { + logger.Fatal(ctx, err) + } + + flagUrl := flag.Arg(0) + + u, err := url.Parse(flagUrl) + if err != nil { + logger.Fatal(ctx, err) + } + + var rev string + if idx := strings.Index(u.Path, "@"); idx > 0 { + rev = u.Path[idx+1:] + } + + cloneOpts := &git.CloneOptions{ + URL: flagUrl, + Progress: os.Stdout, + } + + if len(rev) == 0 { + cloneOpts.SingleBranch = true + cloneOpts.Depth = 1 + } + + if err := cloneOpts.Validate(); err != nil { + logger.Fatal(ctx, err) + } + + repo, err := git.CloneContext(ctx, memory.NewStorage(), nil, cloneOpts) + if err != nil { + logger.Fatal(ctx, err) + } + + ref, err := repo.Head() + if err != nil { + logger.Fatalf(ctx, "failed to get head: %v", err) + } + + commit, err := repo.CommitObject(ref.Hash()) + if err != nil { + logger.Fatalf(ctx, "failed to get commit: %v", err) + } + + tree, err := commit.Tree() + if err != nil { + logger.Fatal(ctx, err) + } + + err = tree.Files().ForEach(func(file *object.File) error { + if file == nil { + return fmt.Errorf("file pointer is nil") + } + + switch file.Mode { + case filemode.Regular: + if strings.HasSuffix(file.Name, "go.mod") { + mvs, err := Direct(file) + if err != nil { + return err + } + Updates(UpdateOptions{ + Pre: false, + Major: false, + Cached: false, + Modules: mvs, + OnUpdate: func(u Update) { + if u.Err != nil { + fmt.Fprintf(os.Stderr, "%s: failed: %v\n", u.Module.Path, u.Err) + } else { + fmt.Printf("%s current:%s => latest:%v\n", u.Module.Path, u.Module.Version, u.Version) + } + }, + }) + } + return nil + default: + return nil + } + }) + + if err != nil { + logger.Fatal(ctx, "failed to exctract file: %v", err) + } +} + +func Direct(file *object.File) ([]module.Version, error) { + r, err := file.Reader() + if err != nil { + return nil, err + } + defer r.Close() + data, err := io.ReadAll(r) + if err != nil { + return nil, err + } + modfile, err := modfile.ParseLax("go.mod", data, nil) + if err != nil { + return nil, err + } + var mods []module.Version + for _, req := range modfile.Require { + // if !req.Indirect { + mods = append(mods, req.Mod) + //} + } + sort.Slice(mods, func(i, j int) bool { + return mods[i].Path < mods[j].Path + }) + return mods, nil +} + +func Periodic(ctx context.Context, db *sqlx.DB) error { + + db. + return nil +} diff --git a/modproxy.go b/modproxy.go new file mode 100644 index 0000000..3ee8ce2 --- /dev/null +++ b/modproxy.go @@ -0,0 +1,292 @@ +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "net/http" + "os" + "path" + "strconv" + "strings" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" + "golang.org/x/sync/errgroup" +) + +// Module contains the module path and versions +type Module struct { + Path string + Versions []string +} + +// MaxVersion returns the latest version. +// If there are no versions, the empty string is returned. +// Prefix can be used to filter the versions based on a prefix. +// If pre is false, pre-release versions will are excluded. +func (m *Module) MaxVersion(prefix string, pre bool) string { + var max string + for _, v := range m.Versions { + if !semver.IsValid(v) || !strings.HasPrefix(v, prefix) { + continue + } + if !pre && semver.Prerelease(v) != "" { + continue + } + max = MaxVersion(v, max) + } + return max +} + +// IsNewerVersion returns true if newversion is greater than oldversion in terms of semver. +// If major is true, then newversion must be a major version ahead of oldversion to be considered newer. +func IsNewerVersion(oldversion, newversion string, major bool) bool { + if major { + return semver.Compare(semver.Major(oldversion), semver.Major(newversion)) < 0 + } + return semver.Compare(oldversion, newversion) < 0 +} + +// MaxVersion returns the larger of two versions according to semantic version precedence. +// Incompatible versions are considered lower than non-incompatible ones. +// Invalid versions are considered lower than valid ones. +// If both versions are invalid, the empty string is returned. +func MaxVersion(v, w string) string { + // sort by validity + vValid := semver.IsValid(v) + wValid := semver.IsValid(w) + if !vValid && !wValid { + return "" + } + if vValid != wValid { + if vValid { + return v + } + return w + } + // sort by compatibility + vIncompatible := strings.HasSuffix(semver.Build(v), "+incompatible") + wIncompatible := strings.HasSuffix(semver.Build(w), "+incompatible") + if vIncompatible != wIncompatible { + if wIncompatible { + return v + } + return w + } + // sort by semver + if semver.Compare(v, w) == 1 { + return v + } + return w +} + +// NextMajor returns the next major version after the provided version +func NextMajor(version string) (string, error) { + major, err := strconv.Atoi(strings.TrimPrefix(semver.Major(version), "v")) + if err != nil { + return "", err + } + major++ + return fmt.Sprintf("v%d", major), nil +} + +// WithMajorPath returns the module path for the provided version +func (m *Module) WithMajorPath(version string) string { + prefix := ModPrefix(m.Path) + return JoinPath(prefix, version, "") +} + +// NextMajorPath returns the module path of the next major version +func (m *Module) NextMajorPath() (string, bool) { + latest := m.MaxVersion("", true) + if latest == "" { + return "", false + } + if semver.Major(latest) == "v0" { + return "", false + } + next, err := NextMajor(latest) + if err != nil { + return "", false + } + return m.WithMajorPath(next), true +} + +// Query the module proxy for all versions of a module. +// If the module does not exist, the second return parameter will be false +// cached sets the Disable-Module-Fetch: true header +func Query(modpath string, cached bool) (*Module, bool, error) { + escaped, err := module.EscapePath(modpath) + if err != nil { + return nil, false, err + } + url := fmt.Sprintf("https://proxy.golang.org/%s/@v/list", escaped) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, false, err + } + req.Header.Set("User-Agent", "GoMajor/1.0") + if cached { + req.Header.Set("Disable-Module-Fetch", "true") + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, false, err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(res.Body) + if res.StatusCode == http.StatusNotFound && bytes.HasPrefix(body, []byte("not found:")) { + return nil, false, nil + } + msg := string(body) + if msg == "" { + msg = res.Status + } + return nil, false, fmt.Errorf("proxy: %s", msg) + } + var mod Module + mod.Path = modpath + sc := bufio.NewScanner(res.Body) + for sc.Scan() { + mod.Versions = append(mod.Versions, sc.Text()) + } + if err := sc.Err(); err != nil { + return nil, false, err + } + return &mod, true, nil +} + +// Latest finds the latest major version of a module +// cached sets the Disable-Module-Fetch: true header +func Latest(modpath string, cached bool) (*Module, error) { + latest, ok, err := Query(modpath, cached) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("module not found: %s", modpath) + } + for i := 0; i < 100; i++ { + nextpath, ok := latest.NextMajorPath() + if !ok { + return latest, nil + } + next, ok, err := Query(nextpath, cached) + if err != nil { + return nil, err + } + if !ok { + // handle the case where a project switched to modules + // without incrementing the major version + version := latest.MaxVersion("", true) + if semver.Build(version) == "+incompatible" { + nextpath = latest.WithMajorPath(semver.Major(version)) + if nextpath != latest.Path { + next, ok, err = Query(nextpath, cached) + if err != nil { + return nil, err + } + } + } + } + if !ok { + return latest, nil + } + latest = next + } + return nil, fmt.Errorf("request limit exceeded") +} + +// QueryPackage tries to find the module path for the provided package path +// it does so by repeatedly chopping off the last path element and trying to +// use it as a path. +func QueryPackage(pkgpath string, cached bool) (*Module, error) { + prefix := pkgpath + for prefix != "" { + if module.CheckPath(prefix) == nil { + mod, ok, err := Query(prefix, cached) + if err != nil { + return nil, err + } + if ok { + modprefix := ModPrefix(mod.Path) + if modpath, pkgdir, ok := SplitPath(modprefix, pkgpath); ok && modpath != mod.Path { + if major, ok := ModMajor(modpath); ok { + if v := mod.MaxVersion(major, false); v != "" { + spec := JoinPath(modprefix, "", pkgdir) + "@" + v + return nil, fmt.Errorf("%s doesn't support import versioning; use %s", major, spec) + } + return nil, fmt.Errorf("failed to find module for package: %s", pkgpath) + } + } + return mod, nil + } + } + remaining, last := path.Split(prefix) + if last == "" { + break + } + prefix = strings.TrimSuffix(remaining, "/") + } + return nil, fmt.Errorf("failed to find module for package: %s", pkgpath) +} + +// Update reports a newer version of a module. +// The Err field will be set if an error occured. +type Update struct { + Module module.Version + Version string + Err error +} + +// UpdateOptions specifies a set of modules to check for updates. +// The OnUpdate callback will be invoked with any updates found. +type UpdateOptions struct { + Pre bool + Cached bool + Major bool + Modules []module.Version + OnUpdate func(Update) +} + +// Updates finds updates for a set of specified modules. +func Updates(opt UpdateOptions) { + ch := make(chan Update) + go func() { + defer close(ch) + private := os.Getenv("GOPRIVATE") + var group errgroup.Group + if opt.Cached { + group.SetLimit(3) + } else { + group.SetLimit(1) + } + for _, m := range opt.Modules { + m := m + if module.MatchPrefixPatterns(private, m.Path) { + continue + } + group.Go(func() error { + mod, err := Latest(m.Path, opt.Cached) + if err != nil { + ch <- Update{Module: m, Err: err} + return nil + } + v := mod.MaxVersion("", opt.Pre) + if IsNewerVersion(m.Version, v, opt.Major) { + ch <- Update{Module: m, Version: v} + } + return nil + }) + } + group.Wait() + }() + for u := range ch { + if opt.OnUpdate != nil { + opt.OnUpdate(u) + } + } +} diff --git a/packages.go b/packages.go new file mode 100644 index 0000000..6ac07b3 --- /dev/null +++ b/packages.go @@ -0,0 +1,115 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +// ModPrefix returns the module path with no SIV +func ModPrefix(modpath string) string { + prefix, _, ok := module.SplitPathVersion(modpath) + if !ok { + prefix = modpath + } + return prefix +} + +// ModMajor returns the major version in vN format +func ModMajor(modpath string) (string, bool) { + _, major, ok := module.SplitPathVersion(modpath) + if ok { + major = strings.TrimPrefix(major, "/") + major = strings.TrimPrefix(major, ".") + } + return major, ok +} + +// SplitPath splits the package path into the module path and the package subdirectory. +// It requires the a module path prefix to figure this out. +func SplitPath(modprefix, pkgpath string) (modpath, pkgdir string, ok bool) { + if !strings.HasPrefix(pkgpath, modprefix) { + return "", "", false + } + modpathlen := len(modprefix) + if rest := pkgpath[modpathlen:]; len(rest) > 0 && rest[0] != '/' && rest[0] != '.' { + return "", "", false + } + if strings.HasPrefix(pkgpath[modpathlen:], "/") { + modpathlen++ + } + if idx := strings.Index(pkgpath[modpathlen:], "/"); idx >= 0 { + modpathlen += idx + } else { + modpathlen = len(pkgpath) + } + modpath = modprefix + if major, ok := ModMajor(pkgpath[:modpathlen]); ok { + modpath = JoinPath(modprefix, major, "") + } + pkgdir = strings.TrimPrefix(pkgpath[len(modpath):], "/") + return modpath, pkgdir, true +} + +// SplitSpec splits the path/to/package@query format strings +func SplitSpec(spec string) (path, query string) { + parts := strings.SplitN(spec, "@", 2) + if len(parts) == 2 { + path = parts[0] + query = parts[1] + } else { + path = spec + } + return +} + +// JoinPath creates a full package path given a module prefix, version, and package directory. +func JoinPath(modprefix, version, pkgdir string) string { + version = strings.TrimPrefix(version, ".") + version = strings.TrimPrefix(version, "/") + major := semver.Major(version) + pkgpath := modprefix + switch { + case strings.HasPrefix(pkgpath, "gopkg.in"): + pkgpath += "." + major + case major != "" && major != "v0" && major != "v1" && !strings.Contains(version, "+incompatible"): + if !strings.HasSuffix(pkgpath, "/") { + pkgpath += "/" + } + pkgpath += major + } + if pkgdir != "" { + pkgpath += "/" + pkgdir + } + return pkgpath +} + +// FindModFile recursively searches up the directory structure until it +// finds the go.mod, reaches the root of the directory tree, or encounters +// an error. +func FindModFile(dir string) (string, error) { + var err error + dir, err = filepath.Abs(dir) + if err != nil { + return "", err + } + for { + name := filepath.Join(dir, "go.mod") + _, err := os.Stat(name) + if err == nil { + return name, nil + } + if !os.IsNotExist(err) { + return "", err + } + parent := filepath.Dir(dir) + if parent == dir { + return "", fmt.Errorf("cannot find go.mod") + } + dir = parent + } +}