summaryrefslogtreecommitdiffstats
path: root/scripts/lib/recipetool/create_go.py
diff options
context:
space:
mode:
authorChristian Lindeberg <christian.lindeberg@axis.com>2025-06-27 14:48:45 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2025-07-01 08:49:37 +0100
commit90cc27f8ce15f2b51de95aa3b6628516030349ee (patch)
treebce4a0782e4588f6e5806c132a92b7ace07ed940 /scripts/lib/recipetool/create_go.py
parent45eb6f8188cc10ad89a3342e925006e9847dd047 (diff)
downloadpoky-90cc27f8ce15f2b51de95aa3b6628516030349ee.tar.gz
recipetool: create_go: Use gomod fetcher instead of go mod vendor
Use the go-mod bbclass together with the gomod fetcher instead of the go-vendor bbclass. (From OE-Core rev: 42b46ab3b92a4f011592e8efcedead075731b8bd) Signed-off-by: Christian Lindeberg <christian.lindeberg@axis.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/recipetool/create_go.py')
-rw-r--r--scripts/lib/recipetool/create_go.py721
1 files changed, 99 insertions, 622 deletions
diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py
index 5cc53931f0..3e9fc85784 100644
--- a/scripts/lib/recipetool/create_go.py
+++ b/scripts/lib/recipetool/create_go.py
@@ -10,48 +10,31 @@
10# 10#
11 11
12 12
13from collections import namedtuple
14from enum import Enum
15from html.parser import HTMLParser
16from recipetool.create import RecipeHandler, handle_license_vars 13from recipetool.create import RecipeHandler, handle_license_vars
17from recipetool.create import find_licenses, tidy_licenses, fixup_license 14from recipetool.create import find_licenses
18from recipetool.create import determine_from_url
19from urllib.error import URLError, HTTPError
20 15
21import bb.utils 16import bb.utils
22import json 17import json
23import logging 18import logging
24import os 19import os
25import re 20import re
26import subprocess
27import sys 21import sys
28import shutil
29import tempfile 22import tempfile
30import urllib.parse 23import urllib.parse
31import urllib.request 24import urllib.request
32 25
33 26
34GoImport = namedtuple('GoImport', 'root vcs url suffix')
35logger = logging.getLogger('recipetool') 27logger = logging.getLogger('recipetool')
36CodeRepo = namedtuple(
37 'CodeRepo', 'path codeRoot codeDir pathMajor pathPrefix pseudoMajor')
38 28
39tinfoil = None 29tinfoil = None
40 30
41# Regular expression to parse pseudo semantic version
42# see https://go.dev/ref/mod#pseudo-versions
43re_pseudo_semver = re.compile(
44 r"^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)(?P<utc>\d{14})-(?P<commithash>[A-Za-z0-9]+)(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$")
45# Regular expression to parse semantic version
46re_semver = re.compile(
47 r"^v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$")
48
49 31
50def tinfoil_init(instance): 32def tinfoil_init(instance):
51 global tinfoil 33 global tinfoil
52 tinfoil = instance 34 tinfoil = instance
53 35
54 36
37
55class GoRecipeHandler(RecipeHandler): 38class GoRecipeHandler(RecipeHandler):
56 """Class to handle the go recipe creation""" 39 """Class to handle the go recipe creation"""
57 40
@@ -83,577 +66,96 @@ class GoRecipeHandler(RecipeHandler):
83 66
84 return bindir 67 return bindir
85 68
86 def __resolve_repository_static(self, modulepath): 69 @staticmethod
87 """Resolve the repository in a static manner 70 def __unescape_path(path):
88 71 """Unescape capital letters using exclamation points."""
89 The method is based on the go implementation of 72 return re.sub(r'!([a-z])', lambda m: m.group(1).upper(), path)
90 `repoRootFromVCSPaths` in
91 https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go
92 """
93
94 url = urllib.parse.urlparse("https://" + modulepath)
95 req = urllib.request.Request(url.geturl())
96
97 try:
98 resp = urllib.request.urlopen(req)
99 # Some modulepath are just redirects to github (or some other vcs
100 # hoster). Therefore, we check if this modulepath redirects to
101 # somewhere else
102 if resp.geturl() != url.geturl():
103 bb.debug(1, "%s is redirectred to %s" %
104 (url.geturl(), resp.geturl()))
105 url = urllib.parse.urlparse(resp.geturl())
106 modulepath = url.netloc + url.path
107
108 except URLError as url_err:
109 # This is probably because the module path
110 # contains the subdir and major path. Thus,
111 # we ignore this error for now
112 logger.debug(
113 1, "Failed to fetch page from [%s]: %s" % (url, str(url_err)))
114
115 host, _, _ = modulepath.partition('/')
116
117 class vcs(Enum):
118 pathprefix = "pathprefix"
119 regexp = "regexp"
120 type = "type"
121 repo = "repo"
122 check = "check"
123 schemelessRepo = "schemelessRepo"
124
125 # GitHub
126 vcsGitHub = {}
127 vcsGitHub[vcs.pathprefix] = "github.com"
128 vcsGitHub[vcs.regexp] = re.compile(
129 r'^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
130 vcsGitHub[vcs.type] = "git"
131 vcsGitHub[vcs.repo] = "https://\\g<root>"
132
133 # Bitbucket
134 vcsBitbucket = {}
135 vcsBitbucket[vcs.pathprefix] = "bitbucket.org"
136 vcsBitbucket[vcs.regexp] = re.compile(
137 r'^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
138 vcsBitbucket[vcs.type] = "git"
139 vcsBitbucket[vcs.repo] = "https://\\g<root>"
140
141 # IBM DevOps Services (JazzHub)
142 vcsIBMDevOps = {}
143 vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git"
144 vcsIBMDevOps[vcs.regexp] = re.compile(
145 r'^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
146 vcsIBMDevOps[vcs.type] = "git"
147 vcsIBMDevOps[vcs.repo] = "https://\\g<root>"
148
149 # Git at Apache
150 vcsApacheGit = {}
151 vcsApacheGit[vcs.pathprefix] = "git.apache.org"
152 vcsApacheGit[vcs.regexp] = re.compile(
153 r'^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
154 vcsApacheGit[vcs.type] = "git"
155 vcsApacheGit[vcs.repo] = "https://\\g<root>"
156
157 # Git at OpenStack
158 vcsOpenStackGit = {}
159 vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org"
160 vcsOpenStackGit[vcs.regexp] = re.compile(
161 r'^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
162 vcsOpenStackGit[vcs.type] = "git"
163 vcsOpenStackGit[vcs.repo] = "https://\\g<root>"
164
165 # chiselapp.com for fossil
166 vcsChiselapp = {}
167 vcsChiselapp[vcs.pathprefix] = "chiselapp.com"
168 vcsChiselapp[vcs.regexp] = re.compile(
169 r'^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$')
170 vcsChiselapp[vcs.type] = "fossil"
171 vcsChiselapp[vcs.repo] = "https://\\g<root>"
172
173 # General syntax for any server.
174 # Must be last.
175 vcsGeneralServer = {}
176 vcsGeneralServer[vcs.regexp] = re.compile(
177 "(?P<root>(?P<repo>([a-z0-9.\\-]+\\.)+[a-z0-9.\\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\\-]+)+?)\\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?(?P<suffix>[A-Za-z0-9_.\\-]+))*$")
178 vcsGeneralServer[vcs.schemelessRepo] = True
179
180 vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps,
181 vcsApacheGit, vcsOpenStackGit, vcsChiselapp,
182 vcsGeneralServer]
183
184 if modulepath.startswith("example.net") or modulepath == "rsc.io":
185 logger.warning("Suspicious module path %s" % modulepath)
186 return None
187 if modulepath.startswith("http:") or modulepath.startswith("https:"):
188 logger.warning("Import path should not start with %s %s" %
189 ("http", "https"))
190 return None
191
192 rootpath = None
193 vcstype = None
194 repourl = None
195 suffix = None
196
197 for srv in vcsPaths:
198 m = srv[vcs.regexp].match(modulepath)
199 if vcs.pathprefix in srv:
200 if host == srv[vcs.pathprefix]:
201 rootpath = m.group('root')
202 vcstype = srv[vcs.type]
203 repourl = m.expand(srv[vcs.repo])
204 suffix = m.group('suffix')
205 break
206 elif m and srv[vcs.schemelessRepo]:
207 rootpath = m.group('root')
208 vcstype = m[vcs.type]
209 repourl = m[vcs.repo]
210 suffix = m.group('suffix')
211 break
212
213 return GoImport(rootpath, vcstype, repourl, suffix)
214
215 def __resolve_repository_dynamic(self, modulepath):
216 """Resolve the repository root in a dynamic manner.
217
218 The method is based on the go implementation of
219 `repoRootForImportDynamic` in
220 https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go
221 """
222 url = urllib.parse.urlparse("https://" + modulepath)
223
224 class GoImportHTMLParser(HTMLParser):
225
226 def __init__(self):
227 super().__init__()
228 self.__srv = {}
229
230 def handle_starttag(self, tag, attrs):
231 if tag == 'meta' and list(
232 filter(lambda a: (a[0] == 'name' and a[1] == 'go-import'), attrs)):
233 content = list(
234 filter(lambda a: (a[0] == 'content'), attrs))
235 if content:
236 srv = content[0][1].split()
237 self.__srv[srv[0]] = srv
238
239 def go_import(self, modulepath):
240 if modulepath in self.__srv:
241 srv = self.__srv[modulepath]
242 return GoImport(srv[0], srv[1], srv[2], None)
243 return None
244 73
245 url = url.geturl() + "?go-get=1" 74 @staticmethod
246 req = urllib.request.Request(url) 75 def __fold_uri(uri):
76 """Fold URI for sorting shorter module paths before longer."""
77 return uri.replace(';', ' ').replace('/', '!')
247 78
248 try: 79 @staticmethod
249 body = urllib.request.urlopen(req).read() 80 def __go_run_cmd(cmd, cwd, d):
250 except HTTPError as http_err: 81 env = dict(os.environ, PATH=d.getVar('PATH'), GOMODCACHE=d.getVar('GOMODCACHE'))
251 logger.warning( 82 return bb.process.run(cmd, env=env, shell=True, cwd=cwd)
252 "Unclean status when fetching page from [%s]: %s", url, str(http_err))
253 body = http_err.fp.read()
254 except URLError as url_err:
255 logger.warning(
256 "Failed to fetch page from [%s]: %s", url, str(url_err))
257 return None
258 83
259 parser = GoImportHTMLParser() 84 def __go_mod(self, go_mod, srctree, localfilesdir, extravalues, d):
260 parser.feed(body.decode('utf-8')) 85 moddir = d.getVar('GOMODCACHE')
261 parser.close()
262 86
263 return parser.go_import(modulepath) 87 # List main packages and their dependencies with the go list command.
88 stdout, _ = self.__go_run_cmd(f"go list -json=Dir,Module -deps {go_mod['Module']['Path']}/...", srctree, d)
89 pkgs = json.loads('[' + stdout.replace('}\n{', '},\n{') + ']')
264 90
265 def __resolve_from_golang_proxy(self, modulepath, version): 91 # Collect licenses for the dependencies.
266 """ 92 licenses = set()
267 Resolves repository data from golang proxy 93 lic_files_chksum = []
268 """ 94 lic_files = {}
269 url = urllib.parse.urlparse("https://proxy.golang.org/" 95 for pkg in pkgs:
270 + modulepath 96 # TODO: If the package is in a subdirectory with its own license
271 + "/@v/" 97 # files then report those istead of the license files found in the
272 + version 98 # module root directory.
273 + ".info") 99 mod = pkg.get('Module', None)
100 if not mod or mod.get('Main', False):
101 continue
102 path = os.path.relpath(mod['Dir'], moddir)
103 for lic in find_licenses(mod['Dir'], d):
104 lic_files[os.path.join(path, lic[1])] = (lic[0], lic[2])
105
106 for lic_file in lic_files:
107 licenses.add(lic_files[lic_file][0])
108 lic_files_chksum.append(
109 f'file://pkg/mod/{lic_file};md5={lic_files[lic_file][1]}')
110
111 # Collect the module cache files downloaded by the go list command as
112 # the go list command knows best what the go list command needs and it
113 # needs more files in the module cache than the go install command as
114 # it doesn't do the dependency pruning mentioned in the Go module
115 # reference, https://go.dev/ref/mod, for go 1.17 or higher.
116 src_uris = []
117 downloaddir = os.path.join(moddir, 'cache', 'download')
118 for dirpath, _, filenames in os.walk(downloaddir):
119 path, base = os.path.split(os.path.relpath(dirpath, downloaddir))
120 if base != '@v':
121 continue
122 path = self.__unescape_path(path)
123 zipver = None
124 for name in filenames:
125 ver, ext = os.path.splitext(name)
126 if ext == '.zip':
127 chksum = bb.utils.sha256_file(os.path.join(dirpath, name))
128 src_uris.append(f'gomod://{path};version={ver};sha256sum={chksum}')
129 zipver = ver
130 break
131 for name in filenames:
132 ver, ext = os.path.splitext(name)
133 if ext == '.mod' and ver != zipver:
134 chksum = bb.utils.sha256_file(os.path.join(dirpath, name))
135 src_uris.append(f'gomod://{path};version={ver};mod=1;sha256sum={chksum}')
274 136
275 # Transform url to lower case, golang proxy doesn't like mixed case 137 self.__go_run_cmd("go clean -modcache", srctree, d)
276 req = urllib.request.Request(url.geturl().lower())
277 138
278 try: 139 licenses_basename = "{pn}-licenses.inc"
279 resp = urllib.request.urlopen(req) 140 licenses_filename = os.path.join(localfilesdir, licenses_basename)
280 except URLError as url_err: 141 with open(licenses_filename, "w") as f:
281 logger.warning( 142 f.write(f'GO_MOD_LICENSES = "{" & ".join(sorted(licenses))}"\n\n')
282 "Failed to fetch page from [%s]: %s", url, str(url_err)) 143 f.write('LIC_FILES_CHKSUM += "\\\n')
283 return None 144 for lic in sorted(lic_files_chksum, key=self.__fold_uri):
145 f.write(' ' + lic + ' \\\n')
146 f.write('"\n')
284 147
285 golang_proxy_res = resp.read().decode('utf-8') 148 extravalues['extrafiles'][f"../{licenses_basename}"] = licenses_filename
286 modinfo = json.loads(golang_proxy_res)
287
288 if modinfo and 'Origin' in modinfo:
289 origin = modinfo['Origin']
290 _root_url = urllib.parse.urlparse(origin['URL'])
291
292 # We normalize the repo URL since we don't want the scheme in it
293 _subdir = origin['Subdir'] if 'Subdir' in origin else None
294 _root, _, _ = self.__split_path_version(modulepath)
295 if _subdir:
296 _root = _root[:-len(_subdir)].strip('/')
297
298 _commit = origin['Hash']
299 _vcs = origin['VCS']
300 return (GoImport(_root, _vcs, _root_url.geturl(), None), _commit)
301
302 return None
303
304 def __resolve_repository(self, modulepath):
305 """
306 Resolves src uri from go module-path
307 """
308 repodata = self.__resolve_repository_static(modulepath)
309 if not repodata or not repodata.url:
310 repodata = self.__resolve_repository_dynamic(modulepath)
311 if not repodata or not repodata.url:
312 logger.error(
313 "Could not resolve repository for module path '%s'" % modulepath)
314 # There is no way to recover from this
315 sys.exit(14)
316 if repodata:
317 logger.debug(1, "Resolved download path for import '%s' => %s" % (
318 modulepath, repodata.url))
319 return repodata
320
321 def __split_path_version(self, path):
322 i = len(path)
323 dot = False
324 for j in range(i, 0, -1):
325 if path[j - 1] < '0' or path[j - 1] > '9':
326 break
327 if path[j - 1] == '.':
328 dot = True
329 break
330 i = j - 1
331
332 if i <= 1 or i == len(
333 path) or path[i - 1] != 'v' or path[i - 2] != '/':
334 return path, "", True
335
336 prefix, pathMajor = path[:i - 2], path[i - 2:]
337 if dot or len(
338 pathMajor) <= 2 or pathMajor[2] == '0' or pathMajor == "/v1":
339 return path, "", False
340
341 return prefix, pathMajor, True
342
343 def __get_path_major(self, pathMajor):
344 if not pathMajor:
345 return ""
346
347 if pathMajor[0] != '/' and pathMajor[0] != '.':
348 logger.error(
349 "pathMajor suffix %s passed to PathMajorPrefix lacks separator", pathMajor)
350
351 if pathMajor.startswith(".v") and pathMajor.endswith("-unstable"):
352 pathMajor = pathMajor[:len("-unstable") - 2]
353
354 return pathMajor[1:]
355
356 def __build_coderepo(self, repo, path):
357 codedir = ""
358 pathprefix, pathMajor, _ = self.__split_path_version(path)
359 if repo.root == path:
360 pathprefix = path
361 elif path.startswith(repo.root):
362 codedir = pathprefix[len(repo.root):].strip('/')
363
364 pseudoMajor = self.__get_path_major(pathMajor)
365
366 logger.debug("root='%s', codedir='%s', prefix='%s', pathMajor='%s', pseudoMajor='%s'",
367 repo.root, codedir, pathprefix, pathMajor, pseudoMajor)
368
369 return CodeRepo(path, repo.root, codedir,
370 pathMajor, pathprefix, pseudoMajor)
371
372 def __resolve_version(self, repo, path, version):
373 hash = None
374 coderoot = self.__build_coderepo(repo, path)
375
376 def vcs_fetch_all():
377 tmpdir = tempfile.mkdtemp()
378 clone_cmd = "%s clone --bare %s %s" % ('git', repo.url, tmpdir)
379 bb.process.run(clone_cmd)
380 log_cmd = "git log --all --pretty='%H %d' --decorate=short"
381 output, _ = bb.process.run(
382 log_cmd, shell=True, stderr=subprocess.PIPE, cwd=tmpdir)
383 bb.utils.prunedir(tmpdir)
384 return output.strip().split('\n')
385
386 def vcs_fetch_remote(tag):
387 # add * to grab ^{}
388 refs = {}
389 ls_remote_cmd = "git ls-remote -q --tags {} {}*".format(
390 repo.url, tag)
391 output, _ = bb.process.run(ls_remote_cmd)
392 output = output.strip().split('\n')
393 for line in output:
394 f = line.split(maxsplit=1)
395 if len(f) != 2:
396 continue
397
398 for prefix in ["HEAD", "refs/heads/", "refs/tags/"]:
399 if f[1].startswith(prefix):
400 refs[f[1][len(prefix):]] = f[0]
401
402 for key, hash in refs.items():
403 if key.endswith(r"^{}"):
404 refs[key.strip(r"^{}")] = hash
405
406 return refs[tag]
407
408 m_pseudo_semver = re_pseudo_semver.match(version)
409
410 if m_pseudo_semver:
411 remote_refs = vcs_fetch_all()
412 short_commit = m_pseudo_semver.group('commithash')
413 for l in remote_refs:
414 r = l.split(maxsplit=1)
415 sha1 = r[0] if len(r) else None
416 if not sha1:
417 logger.error(
418 "Ups: could not resolve abbref commit for %s" % short_commit)
419
420 elif sha1.startswith(short_commit):
421 hash = sha1
422 break
423 else:
424 m_semver = re_semver.match(version)
425 if m_semver:
426
427 def get_sha1_remote(re):
428 rsha1 = None
429 for line in remote_refs:
430 # Split lines of the following format:
431 # 22e90d9b964610628c10f673ca5f85b8c2a2ca9a (tag: sometag)
432 lineparts = line.split(maxsplit=1)
433 sha1 = lineparts[0] if len(lineparts) else None
434 refstring = lineparts[1] if len(
435 lineparts) == 2 else None
436 if refstring:
437 # Normalize tag string and split in case of multiple
438 # regs e.g. (tag: speech/v1.10.0, tag: orchestration/v1.5.0 ...)
439 refs = refstring.strip('(), ').split(',')
440 for ref in refs:
441 if re.match(ref.strip()):
442 rsha1 = sha1
443 return rsha1
444
445 semver = "v" + m_semver.group('major') + "."\
446 + m_semver.group('minor') + "."\
447 + m_semver.group('patch') \
448 + (("-" + m_semver.group('prerelease'))
449 if m_semver.group('prerelease') else "")
450
451 tag = os.path.join(
452 coderoot.codeDir, semver) if coderoot.codeDir else semver
453
454 # probe tag using 'ls-remote', which is faster than fetching
455 # complete history
456 hash = vcs_fetch_remote(tag)
457 if not hash:
458 # backup: fetch complete history
459 remote_refs = vcs_fetch_all()
460 hash = get_sha1_remote(
461 re.compile(fr"(tag:|HEAD ->) ({tag})"))
462
463 logger.debug(
464 "Resolving commit for tag '%s' -> '%s'", tag, hash)
465 return hash
466
467 def __generate_srcuri_inline_fcn(self, path, version, replaces=None):
468 """Generate SRC_URI functions for go imports"""
469
470 logger.info("Resolving repository for module %s", path)
471 # First try to resolve repo and commit from golang proxy
472 # Most info is already there and we don't have to go through the
473 # repository or even perform the version resolve magic
474 golang_proxy_info = self.__resolve_from_golang_proxy(path, version)
475 if golang_proxy_info:
476 repo = golang_proxy_info[0]
477 commit = golang_proxy_info[1]
478 else:
479 # Fallback
480 # Resolve repository by 'hand'
481 repo = self.__resolve_repository(path)
482 commit = self.__resolve_version(repo, path, version)
483
484 url = urllib.parse.urlparse(repo.url)
485 repo_url = url.netloc + url.path
486
487 coderoot = self.__build_coderepo(repo, path)
488
489 inline_fcn = "${@go_src_uri("
490 inline_fcn += f"'{repo_url}','{version}'"
491 if repo_url != path:
492 inline_fcn += f",path='{path}'"
493 if coderoot.codeDir:
494 inline_fcn += f",subdir='{coderoot.codeDir}'"
495 if repo.vcs != 'git':
496 inline_fcn += f",vcs='{repo.vcs}'"
497 if replaces:
498 inline_fcn += f",replaces='{replaces}'"
499 if coderoot.pathMajor:
500 inline_fcn += f",pathmajor='{coderoot.pathMajor}'"
501 inline_fcn += ")}"
502
503 return inline_fcn, commit
504
505 def __go_handle_dependencies(self, go_mod, srctree, localfilesdir, extravalues, d):
506
507 import re
508 src_uris = []
509 src_revs = []
510
511 def generate_src_rev(path, version, commithash):
512 src_rev = f"# {path}@{version} => {commithash}\n"
513 # Ups...maybe someone manipulated the source repository and the
514 # version or commit could not be resolved. This is a sign of
515 # a) the supply chain was manipulated (bad)
516 # b) the implementation for the version resolving didn't work
517 # anymore (less bad)
518 if not commithash:
519 src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
520 src_rev += f"#!!! Could not resolve version !!!\n"
521 src_rev += f"#!!! Possible supply chain attack !!!\n"
522 src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
523 src_rev += f"SRCREV_{path.replace('/', '.')} = \"{commithash}\""
524
525 return src_rev
526
527 # we first go over replacement list, because we are essentialy
528 # interested only in the replaced path
529 if go_mod['Replace']:
530 for replacement in go_mod['Replace']:
531 oldpath = replacement['Old']['Path']
532 path = replacement['New']['Path']
533 version = ''
534 if 'Version' in replacement['New']:
535 version = replacement['New']['Version']
536
537 if os.path.exists(os.path.join(srctree, path)):
538 # the module refers to the local path, remove it from requirement list
539 # because it's a local module
540 go_mod['Require'][:] = [v for v in go_mod['Require'] if v.get('Path') != oldpath]
541 else:
542 # Replace the path and the version, so we don't iterate replacement list anymore
543 for require in go_mod['Require']:
544 if require['Path'] == oldpath:
545 require.update({'Path': path, 'Version': version})
546 break
547
548 for require in go_mod['Require']:
549 path = require['Path']
550 version = require['Version']
551
552 inline_fcn, commithash = self.__generate_srcuri_inline_fcn(
553 path, version)
554 src_uris.append(inline_fcn)
555 src_revs.append(generate_src_rev(path, version, commithash))
556
557 # strip version part from module URL /vXX
558 baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
559 pn, _ = determine_from_url(baseurl)
560 go_mods_basename = "%s-modules.inc" % pn
561 149
150 go_mods_basename = "{pn}-go-mods.inc"
562 go_mods_filename = os.path.join(localfilesdir, go_mods_basename) 151 go_mods_filename = os.path.join(localfilesdir, go_mods_basename)
563 with open(go_mods_filename, "w") as f: 152 with open(go_mods_filename, "w") as f:
564 # We introduce this indirection to make the tests a little easier 153 f.write('SRC_URI += "\\\n')
565 f.write("SRC_URI += \"${GO_DEPENDENCIES_SRC_URI}\"\n") 154 for uri in sorted(src_uris, key=self.__fold_uri):
566 f.write("GO_DEPENDENCIES_SRC_URI = \"\\\n") 155 f.write(' ' + uri + ' \\\n')
567 for uri in src_uris: 156 f.write('"\n')
568 f.write(" " + uri + " \\\n")
569 f.write("\"\n\n")
570 for rev in src_revs:
571 f.write(rev + "\n")
572
573 extravalues['extrafiles'][go_mods_basename] = go_mods_filename
574
575 def __go_run_cmd(self, cmd, cwd, d):
576 return bb.process.run(cmd, env=dict(os.environ, PATH=d.getVar('PATH')),
577 shell=True, cwd=cwd)
578
579 def __go_native_version(self, d):
580 stdout, _ = self.__go_run_cmd("go version", None, d)
581 m = re.match(r".*\sgo((\d+).(\d+).(\d+))\s([\w\/]*)", stdout)
582 major = int(m.group(2))
583 minor = int(m.group(3))
584 patch = int(m.group(4))
585
586 return major, minor, patch
587
588 def __go_mod_patch(self, srctree, localfilesdir, extravalues, d):
589
590 patchfilename = "go.mod.patch"
591 go_native_version_major, go_native_version_minor, _ = self.__go_native_version(
592 d)
593 self.__go_run_cmd("go mod tidy -go=%d.%d" %
594 (go_native_version_major, go_native_version_minor), srctree, d)
595 stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d)
596
597 # Create patch in order to upgrade go version
598 self.__go_run_cmd("git diff go.mod > %s" % (patchfilename), srctree, d)
599 # Restore original state
600 self.__go_run_cmd("git checkout HEAD go.mod go.sum", srctree, d)
601
602 go_mod = json.loads(stdout)
603 tmpfile = os.path.join(localfilesdir, patchfilename)
604 shutil.move(os.path.join(srctree, patchfilename), tmpfile)
605
606 extravalues['extrafiles'][patchfilename] = tmpfile
607
608 return go_mod, patchfilename
609 157
610 def __go_mod_vendor(self, go_mod, srctree, localfilesdir, extravalues, d): 158 extravalues['extrafiles'][f"../{go_mods_basename}"] = go_mods_filename
611 # Perform vendoring to retrieve the correct modules.txt
612 tmp_vendor_dir = tempfile.mkdtemp()
613
614 # -v causes to go to print modules.txt to stderr
615 _, stderr = self.__go_run_cmd(
616 "go mod vendor -v -o %s" % (tmp_vendor_dir), srctree, d)
617
618 modules_txt_basename = "modules.txt"
619 modules_txt_filename = os.path.join(localfilesdir, modules_txt_basename)
620 with open(modules_txt_filename, "w") as f:
621 f.write(stderr)
622
623 extravalues['extrafiles'][modules_txt_basename] = modules_txt_filename
624
625 licenses = []
626 lic_files_chksum = []
627 licvalues = find_licenses(tmp_vendor_dir, d)
628 shutil.rmtree(tmp_vendor_dir)
629
630 if licvalues:
631 for licvalue in licvalues:
632 license = licvalue[0]
633 lics = tidy_licenses(fixup_license(license))
634 lics = [lic for lic in lics if lic not in licenses]
635 if len(lics):
636 licenses.extend(lics)
637 lic_files_chksum.append(
638 'file://src/${GO_IMPORT}/vendor/%s;md5=%s' % (licvalue[1], licvalue[2]))
639
640 # strip version part from module URL /vXX
641 baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
642 pn, _ = determine_from_url(baseurl)
643 licenses_basename = "%s-licenses.inc" % pn
644
645 licenses_filename = os.path.join(localfilesdir, licenses_basename)
646 with open(licenses_filename, "w") as f:
647 f.write("GO_MOD_LICENSES = \"%s\"\n\n" %
648 ' & '.join(sorted(licenses, key=str.casefold)))
649 # We introduce this indirection to make the tests a little easier
650 f.write("LIC_FILES_CHKSUM += \"${VENDORED_LIC_FILES_CHKSUM}\"\n")
651 f.write("VENDORED_LIC_FILES_CHKSUM = \"\\\n")
652 for lic in lic_files_chksum:
653 f.write(" " + lic + " \\\n")
654 f.write("\"\n")
655
656 extravalues['extrafiles'][licenses_basename] = licenses_filename
657 159
658 def process(self, srctree, classes, lines_before, 160 def process(self, srctree, classes, lines_before,
659 lines_after, handled, extravalues): 161 lines_after, handled, extravalues):
@@ -672,56 +174,30 @@ class GoRecipeHandler(RecipeHandler):
672 174
673 d.prependVar('PATH', '%s:' % go_bindir) 175 d.prependVar('PATH', '%s:' % go_bindir)
674 handled.append('buildsystem') 176 handled.append('buildsystem')
675 classes.append("go-vendor") 177 classes.append("go-mod")
676 178
677 stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d) 179 tmp_mod_dir = tempfile.mkdtemp(prefix='go-mod-')
180 d.setVar('GOMODCACHE', tmp_mod_dir)
678 181
182 stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d)
679 go_mod = json.loads(stdout) 183 go_mod = json.loads(stdout)
680 go_import = go_mod['Module']['Path'] 184 go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path'])
681 go_version_match = re.match("([0-9]+).([0-9]+)", go_mod['Go'])
682 go_version_major = int(go_version_match.group(1))
683 go_version_minor = int(go_version_match.group(2))
684 src_uris = []
685 185
686 localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-') 186 localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
687 extravalues.setdefault('extrafiles', {}) 187 extravalues.setdefault('extrafiles', {})
688 188
689 # Use an explicit name determined from the module name because it 189 # Write the ${BPN}-licenses.inc and ${BPN}-go-mods.inc files
690 # might differ from the actual URL for replaced modules 190 self.__go_mod(go_mod, srctree, localfilesdir, extravalues, d)
691 # strip version part from module URL /vXX
692 baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
693 pn, _ = determine_from_url(baseurl)
694
695 # go.mod files with version < 1.17 may not include all indirect
696 # dependencies. Thus, we have to upgrade the go version.
697 if go_version_major == 1 and go_version_minor < 17:
698 logger.warning(
699 "go.mod files generated by Go < 1.17 might have incomplete indirect dependencies.")
700 go_mod, patchfilename = self.__go_mod_patch(srctree, localfilesdir,
701 extravalues, d)
702 src_uris.append(
703 "file://%s;patchdir=src/${GO_IMPORT}" % (patchfilename))
704
705 # Check whether the module is vendored. If so, we have nothing to do.
706 # Otherwise we gather all dependencies and add them to the recipe
707 if not os.path.exists(os.path.join(srctree, "vendor")):
708
709 # Write additional $BPN-modules.inc file
710 self.__go_mod_vendor(go_mod, srctree, localfilesdir, extravalues, d)
711 lines_before.append("LICENSE += \" & ${GO_MOD_LICENSES}\"")
712 lines_before.append("require %s-licenses.inc" % (pn))
713
714 self.__rewrite_src_uri(lines_before, ["file://modules.txt"])
715
716 self.__go_handle_dependencies(go_mod, srctree, localfilesdir, extravalues, d)
717 lines_before.append("require %s-modules.inc" % (pn))
718 191
719 # Do generic license handling 192 # Do generic license handling
720 handle_license_vars(srctree, lines_before, handled, extravalues, d) 193 handle_license_vars(srctree, lines_before, handled, extravalues, d)
721 self.__rewrite_lic_uri(lines_before) 194 self.__rewrite_lic_vars(lines_before)
195
196 self.__rewrite_src_uri(lines_before)
722 197
723 lines_before.append("GO_IMPORT = \"{}\"".format(baseurl)) 198 lines_before.append('require ${BPN}-licenses.inc')
724 lines_before.append("SRCREV_FORMAT = \"${BPN}\"") 199 lines_before.append('require ${BPN}-go-mods.inc')
200 lines_before.append(f'GO_IMPORT = "{go_import}"')
725 201
726 def __update_lines_before(self, updated, newlines, lines_before): 202 def __update_lines_before(self, updated, newlines, lines_before):
727 if updated: 203 if updated:
@@ -733,9 +209,11 @@ class GoRecipeHandler(RecipeHandler):
733 lines_before.append(line) 209 lines_before.append(line)
734 return updated 210 return updated
735 211
736 def __rewrite_lic_uri(self, lines_before): 212 def __rewrite_lic_vars(self, lines_before):
737 213
738 def varfunc(varname, origvalue, op, newlines): 214 def varfunc(varname, origvalue, op, newlines):
215 if varname == 'LICENSE':
216 return ' & '.join((origvalue, '${GO_MOD_LICENSES}')), None, -1, True
739 if varname == 'LIC_FILES_CHKSUM': 217 if varname == 'LIC_FILES_CHKSUM':
740 new_licenses = [] 218 new_licenses = []
741 licenses = origvalue.split('\\') 219 licenses = origvalue.split('\\')
@@ -757,15 +235,14 @@ class GoRecipeHandler(RecipeHandler):
757 return origvalue, None, 0, True 235 return origvalue, None, 0, True
758 236
759 updated, newlines = bb.utils.edit_metadata( 237 updated, newlines = bb.utils.edit_metadata(
760 lines_before, ['LIC_FILES_CHKSUM'], varfunc) 238 lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc)
761 return self.__update_lines_before(updated, newlines, lines_before) 239 return self.__update_lines_before(updated, newlines, lines_before)
762 240
763 def __rewrite_src_uri(self, lines_before, additional_uris = []): 241 def __rewrite_src_uri(self, lines_before):
764 242
765 def varfunc(varname, origvalue, op, newlines): 243 def varfunc(varname, origvalue, op, newlines):
766 if varname == 'SRC_URI': 244 if varname == 'SRC_URI':
767 src_uri = ["git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https"] 245 src_uri = ['git://${GO_IMPORT};protocol=https;nobranch=1;destsuffix=${GO_SRCURI_DESTSUFFIX}']
768 src_uri.extend(additional_uris)
769 return src_uri, None, -1, True 246 return src_uri, None, -1, True
770 return origvalue, None, 0, True 247 return origvalue, None, 0, True
771 248