1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
|
# Recipe creation tool - go support plugin
#
# The code is based on golang internals. See the afftected
# methods for further reference and information.
#
# Copyright (C) 2023 Weidmueller GmbH & Co KG
# Author: Lukas Funke <lukas.funke@weidmueller.com>
#
# SPDX-License-Identifier: GPL-2.0-only
#
from recipetool.create import RecipeHandler, handle_license_vars
from recipetool.create import find_licenses
import bb.utils
import json
import logging
import os
import re
import sys
import tempfile
import urllib.parse
import urllib.request
logger = logging.getLogger('recipetool')
tinfoil = None
def tinfoil_init(instance):
global tinfoil
tinfoil = instance
class GoRecipeHandler(RecipeHandler):
"""Class to handle the go recipe creation"""
@staticmethod
def __ensure_go():
"""Check if the 'go' command is available in the recipes"""
recipe = "go-native"
if not tinfoil.recipes_parsed:
tinfoil.parse_recipes()
try:
rd = tinfoil.parse_recipe(recipe)
except bb.providers.NoProvider:
bb.error(
"Nothing provides '%s' which is required for the build" % (recipe))
bb.note(
"You will likely need to add a layer that provides '%s'" % (recipe))
return None
bindir = rd.getVar('STAGING_BINDIR_NATIVE')
gopath = os.path.join(bindir, 'go')
if not os.path.exists(gopath):
tinfoil.build_targets(recipe, 'addto_recipe_sysroot')
if not os.path.exists(gopath):
logger.error(
'%s required to process specified source, but %s did not seem to populate it' % 'go', recipe)
return None
return bindir
@staticmethod
def __unescape_path(path):
"""Unescape capital letters using exclamation points."""
return re.sub(r'!([a-z])', lambda m: m.group(1).upper(), path)
@staticmethod
def __fold_uri(uri):
"""Fold URI for sorting shorter module paths before longer."""
return uri.replace(';', ' ').replace('/', '!')
@staticmethod
def __go_run_cmd(cmd, cwd, d):
env = dict(os.environ, PATH=d.getVar('PATH'), GOMODCACHE=d.getVar('GOMODCACHE'))
return bb.process.run(cmd, env=env, shell=True, cwd=cwd)
def __go_mod(self, go_mod, srctree, localfilesdir, extravalues, d):
moddir = d.getVar('GOMODCACHE')
# List main packages and their dependencies with the go list command.
stdout, _ = self.__go_run_cmd(f"go list -json=Dir,Module -deps {go_mod['Module']['Path']}/...", srctree, d)
pkgs = json.loads('[' + stdout.replace('}\n{', '},\n{') + ']')
# Collect licenses for the dependencies.
licenses = set()
lic_files_chksum = []
lic_files = {}
for pkg in pkgs:
# TODO: If the package is in a subdirectory with its own license
# files then report those istead of the license files found in the
# module root directory.
mod = pkg.get('Module', None)
if not mod or mod.get('Main', False):
continue
path = os.path.relpath(mod['Dir'], moddir)
for lic in find_licenses(mod['Dir'], d):
lic_files[os.path.join(path, lic[1])] = (lic[0], lic[2])
for lic_file in lic_files:
licenses.add(lic_files[lic_file][0])
lic_files_chksum.append(
f'file://pkg/mod/{lic_file};md5={lic_files[lic_file][1]}')
# Collect the module cache files downloaded by the go list command as
# the go list command knows best what the go list command needs and it
# needs more files in the module cache than the go install command as
# it doesn't do the dependency pruning mentioned in the Go module
# reference, https://go.dev/ref/mod, for go 1.17 or higher.
src_uris = []
downloaddir = os.path.join(moddir, 'cache', 'download')
for dirpath, _, filenames in os.walk(downloaddir):
path, base = os.path.split(os.path.relpath(dirpath, downloaddir))
if base != '@v':
continue
path = self.__unescape_path(path)
zipver = None
for name in filenames:
ver, ext = os.path.splitext(name)
if ext == '.zip':
chksum = bb.utils.sha256_file(os.path.join(dirpath, name))
src_uris.append(f'gomod://{path};version={ver};sha256sum={chksum}')
zipver = ver
break
for name in filenames:
ver, ext = os.path.splitext(name)
if ext == '.mod' and ver != zipver:
chksum = bb.utils.sha256_file(os.path.join(dirpath, name))
src_uris.append(f'gomod://{path};version={ver};mod=1;sha256sum={chksum}')
self.__go_run_cmd("go clean -modcache", srctree, d)
licenses_basename = "{pn}-licenses.inc"
licenses_filename = os.path.join(localfilesdir, licenses_basename)
with open(licenses_filename, "w") as f:
f.write(f'GO_MOD_LICENSES = "{" & ".join(sorted(licenses))}"\n\n')
f.write('LIC_FILES_CHKSUM += "\\\n')
for lic in sorted(lic_files_chksum, key=self.__fold_uri):
f.write(' ' + lic + ' \\\n')
f.write('"\n')
extravalues['extrafiles'][f"../{licenses_basename}"] = licenses_filename
go_mods_basename = "{pn}-go-mods.inc"
go_mods_filename = os.path.join(localfilesdir, go_mods_basename)
with open(go_mods_filename, "w") as f:
f.write('SRC_URI += "\\\n')
for uri in sorted(src_uris, key=self.__fold_uri):
f.write(' ' + uri + ' \\\n')
f.write('"\n')
extravalues['extrafiles'][f"../{go_mods_basename}"] = go_mods_filename
def process(self, srctree, classes, lines_before,
lines_after, handled, extravalues):
if 'buildsystem' in handled:
return False
files = RecipeHandler.checkfiles(srctree, ['go.mod'])
if not files:
return False
d = bb.data.createCopy(tinfoil.config_data)
go_bindir = self.__ensure_go()
if not go_bindir:
sys.exit(14)
d.prependVar('PATH', '%s:' % go_bindir)
handled.append('buildsystem')
classes.append("go-mod")
tmp_mod_dir = tempfile.mkdtemp(prefix='go-mod-')
d.setVar('GOMODCACHE', tmp_mod_dir)
stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d)
go_mod = json.loads(stdout)
go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path'])
localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
extravalues.setdefault('extrafiles', {})
# Write the ${BPN}-licenses.inc and ${BPN}-go-mods.inc files
self.__go_mod(go_mod, srctree, localfilesdir, extravalues, d)
# Do generic license handling
handle_license_vars(srctree, lines_before, handled, extravalues, d)
self.__rewrite_lic_vars(lines_before)
self.__rewrite_src_uri(lines_before)
lines_before.append('require ${BPN}-licenses.inc')
lines_before.append('require ${BPN}-go-mods.inc')
lines_before.append(f'GO_IMPORT = "{go_import}"')
def __update_lines_before(self, updated, newlines, lines_before):
if updated:
del lines_before[:]
for line in newlines:
# Hack to avoid newlines that edit_metadata inserts
if line.endswith('\n'):
line = line[:-1]
lines_before.append(line)
return updated
def __rewrite_lic_vars(self, lines_before):
def varfunc(varname, origvalue, op, newlines):
if varname == 'LICENSE':
return ' & '.join((origvalue, '${GO_MOD_LICENSES}')), None, -1, True
if varname == 'LIC_FILES_CHKSUM':
new_licenses = []
licenses = origvalue.split('\\')
for license in licenses:
if not license:
logger.warning("No license file was detected for the main module!")
# the license list of the main recipe must be empty
# this can happen for example in case of CLOSED license
# Fall through to complete recipe generation
continue
license = license.strip()
uri, chksum = license.split(';', 1)
url = urllib.parse.urlparse(uri)
new_uri = os.path.join(
url.scheme + "://", "src", "${GO_IMPORT}", url.netloc + url.path) + ";" + chksum
new_licenses.append(new_uri)
return new_licenses, None, -1, True
return origvalue, None, 0, True
updated, newlines = bb.utils.edit_metadata(
lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc)
return self.__update_lines_before(updated, newlines, lines_before)
def __rewrite_src_uri(self, lines_before):
def varfunc(varname, origvalue, op, newlines):
if varname == 'SRC_URI':
src_uri = ['git://${GO_IMPORT};protocol=https;nobranch=1;destsuffix=${GO_SRCURI_DESTSUFFIX}']
return src_uri, None, -1, True
return origvalue, None, 0, True
updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
return self.__update_lines_before(updated, newlines, lines_before)
def register_recipe_handlers(handlers):
handlers.append((GoRecipeHandler(), 60))
|
