MissingRemoteIdCheck: check for missing remote-id

Scans HOMEPAGE and SRC_URI for uris matching regexes using which it
extracts remote-id. Skips already defined remote-id types. Skips URIS
that end with ".diff" or ".patch". Prefers to take remote-id from newer
package versions, in case URL updated.

Resolves: https://github.com/pkgcore/pkgcheck/issues/475
Signed-off-by: Arthur Zamarin <arthurzam@gentoo.org>

12 files changed, 92 insertions, 2 deletions

diff --git a/testdata/data/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/expected.json b/testdata/data/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/expected.json
new file mode 100644
index 00000000..ab5ae2d4
--- /dev/null
+++ b/testdata/data/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/expected.json
@@ -0,0 +1,4 @@
+{"__class__": "MissingRemoteId", "category": "MissingRemoteIdCheck", "package": "MissingRemoteId", "remote_type": "gitlab", "value": "pkgcore/pkgcheck/extra/MissingRemoteId", "uri": "https://gitlab.com/pkgcore/pkgcheck/extra/MissingRemoteId/-/archive/1/MissingRemoteId-1.tar.bz2"}
+{"__class__": "MissingRemoteId", "category": "MissingRemoteIdCheck", "package": "MissingRemoteId", "remote_type": "heptapod", "value": "pkgcore/pkgcore", "uri": "https://foss.heptapod.net/pkgcore/pkgcore/-/archive/4/MissingRemoteId-4.tar.bz2"}
+{"__class__": "MissingRemoteId", "category": "MissingRemoteIdCheck", "package": "MissingRemoteId", "remote_type": "pypi", "value": "MissingRemoteId", "uri": "https://files.pythonhosted.org/packages/source/M/MissingRemoteId/MissingRemoteId-1.tar.gz"}
+{"__class__": "MissingRemoteId", "category": "MissingRemoteIdCheck", "package": "MissingRemoteId", "remote_type": "sourceforge", "value": "pkgcheck", "uri": "https://pkgcheck.sourceforge.net/"}
diff --git a/testdata/data/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/fix.patch b/testdata/data/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/fix.patch
new file mode 100644
index 00000000..035f9ad4
--- /dev/null
+++ b/testdata/data/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/fix.patch
@@ -0,0 +1,13 @@
+--- eapis-testing/MissingRemoteIdCheck/MissingRemoteId/metadata.xml
++++ fixed/MissingRemoteIdCheck/MissingRemoteId/metadata.xml
+@@ -3,6 +3,10 @@
+ <pkgmetadata>
+ 	<upstream>
+ 		<remote-id type="bitbucket">pkgcore/pkgcheck</remote-id>
++		<remote-id type="gitlab">pkgcore/pkgcheck/extra/MissingRemoteId</remote-id>
++		<remote-id type="heptapod">pkgcore/pkgcheck</remote-id>
++		<remote-id type="pypi">MissingRemoteId</remote-id>
++		<remote-id type="sourceforge">pkgcheck</remote-id>
+ 	</upstream>
+ 	<use>
+ 		<flag name="test">enable tests</flag>
diff --git a/testdata/repos/eapis-testing/EapiCheck/StableKeywordsOnTestingEapi/StableKeywordsOnTestingEapi-0.ebuild b/testdata/repos/eapis-testing/EapiCheck/StableKeywordsOnTestingEapi/StableKeywordsOnTestingEapi-0.ebuild
index cd015c21..dddf98ef 100644
--- a/testdata/repos/eapis-testing/EapiCheck/StableKeywordsOnTestingEapi/StableKeywordsOnTestingEapi-0.ebuild
+++ b/testdata/repos/eapis-testing/EapiCheck/StableKeywordsOnTestingEapi/StableKeywordsOnTestingEapi-0.ebuild
@@ -1,7 +1,7 @@
 EAPI=7
 
 DESCRIPTION="Ebuild with stable keywords on EAPI marked as stable"
-HOMEPAGE="https://github.com/pkgcore/pkgcheck"
+HOMEPAGE="https://pkgcore.github.io/pkgcheck/"
 SRC_URI=""
 
 LICENSE="BSD"
diff --git a/testdata/repos/eapis-testing/EapiCheck/StableKeywordsOnTestingEapi/StableKeywordsOnTestingEapi-1.ebuild b/testdata/repos/eapis-testing/EapiCheck/StableKeywordsOnTestingEapi/StableKeywordsOnTestingEapi-1.ebuild
index 5ae1a8d9..40751969 100644
--- a/testdata/repos/eapis-testing/EapiCheck/StableKeywordsOnTestingEapi/StableKeywordsOnTestingEapi-1.ebuild
+++ b/testdata/repos/eapis-testing/EapiCheck/StableKeywordsOnTestingEapi/StableKeywordsOnTestingEapi-1.ebuild
@@ -1,7 +1,7 @@
 EAPI=8
 
 DESCRIPTION="Ebuild with stable keywords on EAPI marked as testing"
-HOMEPAGE="https://github.com/pkgcore/pkgcheck"
+HOMEPAGE="https://pkgcore.github.io/pkgcheck/"
 SRC_URI=""
 
 LICENSE="BSD"
diff --git a/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/Manifest b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/Manifest
new file mode 100644
index 00000000..112fbef4
--- /dev/null
+++ b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/Manifest
@@ -0,0 +1,8 @@
+DIST MissingRemoteId-0.gh.tar.gz 474194 BLAKE2B 8abccad85b39a1c3125ba6685f568892e395316138460b7c6d09ce781f4565f66e812a730e1ab5ae7c9bf83bd3bb9e088623036ae12a9f17f54f8fc31b019ecd SHA512 be433a54fa5cf4c421e74af0d8556ce4f666089260aaecc280f53e4cafbdea2b97735e25d22af0c44928b0cba57ce19aaa7d3a25e23ac0d87624e596ddb27e34
+DIST MissingRemoteId-1-extra.tar.bz2 471942 BLAKE2B 534a32f107067f4237da2cc396d65f42d81f6b8b3bf09abbb84ffd6a33542374d66a15047bb3428c12e4debc771a5661ba3db42c8b7009914815d6c137673b4b SHA512 078310c9cc0154f1ffec6c6c25b3d301a644cc2163cc1dadb24a5f77a4c3d89583dfc7f44972078d59393cdac028bfeb3b481ab03fe9f5a3ebfd6bb0f5ba3a73
+DIST MissingRemoteId-1.tar.gz 497163 BLAKE2B 600b51c5c034356fcfbbbb741681e7e304ccf14b9390207305d35c48dff7675e808101fa76f6a4ce250691b2fd99deae7d59cc91560d609c1a2c3d7421859849 SHA512 3539d877b63e739877f79340ff59c3592f43ac9e00b507ab63d7afc2bb3eaa32f863e7cc8029c5a05bcf7068de70fd3149447054f0d9d304d4e3548a0a25d676
+DIST MissingRemoteId-2.tar.gz 467493 BLAKE2B ecfc1e435397f52cc174724b08e806ee92b49438d05f28fb9a698d2a5778b8fae3a5aa8cfa574512725d34a364774722ddb38689d7db13cd2fcfccc7451f69f1 SHA512 15da0fb2c3615739a8ebd268e1b7a924e24c43f1c7390a78793b8345480a7a23718f281616dbabc9510ac002005c27ec3c64743368593b9413198ae0718395ff
+DIST MissingRemoteId-3.tar.bz2 469597 BLAKE2B 6f2234da65c9903dd1c7727e163f05965ebdbb303cfab162bc0070a79507981e2b88917e0d3eaab20d352c6f369d71acba195a8cc0c5f6ae28252b1e369658c7 SHA512 a40641bf6dba77b6b98e20d95ff6f92a48a500f737e55ef1d500c4469bb8eb489af83c84264731dac0654caadbd520105ef85d669fe01930f507764750f44f35
+DIST MissingRemoteId-4.tar.bz2 467403 BLAKE2B e31bdf219d4aa01c2255f1a6b29e88457b9ea7bb078c5eb797e100662e0635b6eb28d15c411aaa3e676fd0a8fb929f9e50018bbcaa1856999469e2b06ad2b43c SHA512 b5d3978acce3b03f2b46fc2e7bf5e61185bfb745835a375d7d1d76bb210ef7f661e0a938e1510a1f9b815d9fe832f318f2f23d23040a21782a2502a473c561fd
+DIST MissingRemoteId-486.diff 4480 BLAKE2B 9fd62254dd8a4f0cbbaaf7edde1eea013e6b06cacf90b605cda8fea8afac678a13450854a5a8b0a37e730675e31229edb2be1d30756e87ee0de3af5334ba74b5 SHA512 ba45d0eb7bb47eaddf950ff535e1cf448bfcf5528615efc214c71c8489a0578871763e6909327f2d86784905cfe78d15441991615c59c97e887c3a71fce80094
+DIST MissingRemoteId-486.patch 14111416 BLAKE2B fd3ac79ffe3ca031d40a6523a4ae900200c1b1eea0369fd1eec3233f852af8edffd81ca6c0bba628132221346d283d2138794e1f9697074fc1044944ad852e5b SHA512 a2cce0b1fd2267cc8dbd3e2ad3ec1f99526de3988443c0937405cc7b2eab9ef517001ef0b4b387e8721d3c33610cd642c6b7cf83217996a83481830de9713de7
diff --git a/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-0.ebuild b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-0.ebuild
new file mode 100644
index 00000000..5b605897
--- /dev/null
+++ b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-0.ebuild
@@ -0,0 +1,16 @@
+EAPI=7
+
+DESCRIPTION="Skip patch and diff urls, and urls behind use flags"
+HOMEPAGE="https://pkgcore.github.io/pkgcheck/"
+SRC_URI="
+	https://github.com/pkgcore/pkgcheck/pull/486.patch -> ${PN}-486.patch
+	https://github.com/pkgcore/pkgcheck/pull/486.diff -> ${PN}-486.diff
+	test? (
+		https://github.com/pkgcore/pkgcheck/archive/v${PV}.tar.gz
+		 -> ${P}.gh.tar.gz
+	)
+"
+LICENSE="BSD"
+SLOT="0"
+IUSE="test"
+RESTRICT="!test? ( test )"
diff --git a/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-1.ebuild b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-1.ebuild
new file mode 100644
index 00000000..2a49ad82
--- /dev/null
+++ b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-1.ebuild
@@ -0,0 +1,10 @@
+EAPI=7
+
+DESCRIPTION="Use from mirror://pypi and gitlab archive"
+HOMEPAGE="https://pkgcore.github.io/pkgcheck/"
+SRC_URI="
+	mirror://pypi/${PN::1}/${PN}/${P}.tar.gz
+	https://gitlab.com/pkgcore/pkgcheck/extra/${PN}/-/archive/${PV}/${P}.tar.bz2 -> ${P}-extra.tar.bz2
+"
+LICENSE="BSD"
+SLOT="0"
diff --git a/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-2.ebuild b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-2.ebuild
new file mode 100644
index 00000000..ed226b8e
--- /dev/null
+++ b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-2.ebuild
@@ -0,0 +1,7 @@
+EAPI=7
+
+DESCRIPTION="Check homepage"
+HOMEPAGE="https://pkgcheck.sourceforge.net/"
+SRC_URI="mirror://sourceforge/${PN}/${P}.tar.gz"
+LICENSE="BSD"
+SLOT="0"
diff --git a/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-3.ebuild b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-3.ebuild
new file mode 100644
index 00000000..e9a20138
--- /dev/null
+++ b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-3.ebuild
@@ -0,0 +1,9 @@
+EAPI=7
+
+DESCRIPTION="Don't suggest where already value exists"
+HOMEPAGE="https://bitbucket.org/pkgcore/pkgcore"
+SRC_URI="
+	https://bitbucket.org/pkgcore/pkgdev/get/v${PV}.tar.bz2 -> ${P}.tar.bz2
+"
+LICENSE="BSD"
+SLOT="0"
diff --git a/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-4.ebuild b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-4.ebuild
new file mode 100644
index 00000000..970d959e
--- /dev/null
+++ b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/MissingRemoteId-4.ebuild
@@ -0,0 +1,11 @@
+EAPI=7
+
+DESCRIPTION="Don't suggest where already value exists"
+HOMEPAGE="https://pkgcore.github.io/pkgcheck/"
+SRC_URI="
+	https://foss.heptapod.net/pkgcore/pkgcheck/-/archive/${PV}/${P}.tar.bz2
+	https://foss.heptapod.net/pkgcore/pkgcheck/extra/-/archive/${PV}/${P}.tar.bz2
+	https://foss.heptapod.net/pkgcore/pkgcore/-/archive/${PV}/${P}.tar.bz2
+"
+LICENSE="BSD"
+SLOT="0"
diff --git a/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/metadata.xml b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/metadata.xml
new file mode 100644
index 00000000..2048587f
--- /dev/null
+++ b/testdata/repos/eapis-testing/MissingRemoteIdCheck/MissingRemoteId/metadata.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE pkgmetadata SYSTEM "https://www.gentoo.org/dtd/metadata.dtd">
+<pkgmetadata>
+	<upstream>
+		<remote-id type="bitbucket">pkgcore/pkgcheck</remote-id>
+	</upstream>
+	<use>
+		<flag name="test">enable tests</flag>
+	</use>
+</pkgmetadata>
diff --git a/testdata/repos/eapis-testing/profiles/thirdpartymirrors b/testdata/repos/eapis-testing/profiles/thirdpartymirrors
new file mode 100644
index 00000000..15399f5c
--- /dev/null
+++ b/testdata/repos/eapis-testing/profiles/thirdpartymirrors
@@ -0,0 +1,2 @@
+pypi            https://files.pythonhosted.org/packages/source
+sourceforge     https://downloads.sourceforge.net