1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
|
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
"""
BitBake 'Fetch' implementation for cargo registries, e.g. crates.io
SRC_URI = "crate://some.host/crate_name/version;OptionA=xxx;OptionB=xxx;..."
Supported SRC_URI options are:
- protocol
The method to use to download the crate file. Common options are "http",
"https", and "ftp". The default is "https".
"""
# Copyright (C) 2016 Doug Goldstein
#
# SPDX-License-Identifier: GPL-2.0-only
#
# Based on functions from the base bb module, Copyright 2003 Holger Schurig
import hashlib
import json
import os
import subprocess
import re
from functools import cmp_to_key
import bb
from bb.fetch import logger, subprocess_setup, UnpackError, runfetchcmd
from bb.fetch.wget import Wget
class Crate(Wget):
"""Class to fetch crates via wget"""
def _cargo_bitbake_path(self, rootdir):
return os.path.join(rootdir, "cargo_home", "bitbake")
def supports(self, ud, d):
"""
Check to see if a given url is for this fetcher
"""
return ud.type in ['crate']
def recommends_checksum(self, urldata):
return True
def urldata_init(self, ud, d):
"""
Sets up to download the respective crate from crates.io
"""
if ud.type == 'crate':
self._crate_urldata_init(ud, d)
super(Crate, self).urldata_init(ud, d)
def _generate_index_path(self, name):
# https://doc.rust-lang.org/cargo/reference/registry-index.html#index-files
if len(name) == 1:
return f"1/{name}"
elif len(name) == 2:
return f"2/{name}"
elif len(name) == 3:
return f"3/{name[0]}/{name}"
else:
return f"{name[0:2]}/{name[2:4]}/{name}"
def _crate_urldata_init(self, ud, d):
"""
Sets up the download for a crate
"""
# URL syntax is: crate://HOST/NAME/VERSION
# break the URL apart by /
parts = ud.url.split('/')
if len(parts) < 5:
raise bb.fetch.ParameterError("Invalid URL: Must be crate://HOST/NAME/VERSION", ud.url)
# version is expected to be the last token
# but ignore possible url parameters which will be used
# by the top fetcher class
version = parts[-1].split(";")[0]
# second to last field is name
name = parts[-2]
# host (this is to allow custom crate registries to be specified
host = '/'.join(parts[2:-2])
if 'protocol' in ud.parm:
proto = ud.parm['protocol']
else:
proto = 'https'
# If using crates.io use the CDN directly as per https://crates.io/data-access
if host == 'crates.io':
if proto != 'https':
bb.warn("URL: %s does the %s protocol which is not supported by crates.io. Please change to ;protocol=https in the url." % (ud.url, proto))
ud.url = "https://static.crates.io/crates/%s/%s/download" % (name, version)
ud.versionsurl = 'https://index.crates.io/' + self._generate_index_path(name)
else:
ud.url = "%s://%s/%s/%s/download" % (proto, host, name, version)
ud.versionsurl = "%s://%s/%s/versions" % (proto, host, name)
ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
if 'name' not in ud.parm:
ud.parm['name'] = '%s-%s' % (name, version)
logger.debug2("Fetching %s to %s" % (ud.url, ud.parm['downloadfilename']))
def unpack(self, ud, rootdir, d):
"""
Uses the crate to build the necessary paths for cargo to utilize it
"""
if ud.type == 'crate':
return self._crate_unpack(ud, rootdir, d)
else:
super(Crate, self).unpack(ud, rootdir, d)
def _crate_unpack(self, ud, rootdir, d):
"""
Unpacks a crate
"""
thefile = ud.localpath
# possible metadata we need to write out
metadata = {}
bp = d.getVar('BP')
if bp == ud.parm.get('name'):
cmd = ['tar', '-xz', '--no-same-owner', '-f', thefile]
ud.unpack_tracer.unpack("crate-extract", rootdir)
else:
cargo_bitbake = self._cargo_bitbake_path(rootdir)
ud.unpack_tracer.unpack("cargo-extract", cargo_bitbake)
cmd = ['tar', '-xz', '--no-same-owner', '-f', thefile, '-C', cargo_bitbake]
# ensure we've got these paths made
bb.utils.mkdirhier(cargo_bitbake)
# generate metadata necessary
with open(thefile, 'rb') as f:
# get the SHA256 of the original tarball
tarhash = hashlib.sha256(f.read()).hexdigest()
metadata['files'] = {}
metadata['package'] = tarhash
bb.note("Unpacking %s to %s/" % (thefile, os.getcwd()))
runfetchcmd(cmd, d, workdir=rootdir)
# if we have metadata to write out..
if len(metadata) > 0:
cratepath = os.path.splitext(os.path.basename(thefile))[0]
bbpath = self._cargo_bitbake_path(rootdir)
mdfile = '.cargo-checksum.json'
mdpath = os.path.join(bbpath, cratepath, mdfile)
with open(mdpath, "w") as f:
json.dump(metadata, f)
def latest_versionstring(self, ud, d, filter_regex=None):
"""
Return the latest upstream version, dispatching to the appropriate
parser based on the versionsurl format.
"""
if ud.versionsurl.startswith('https://index.crates.io/'):
return self._latest_versionstring_from_index(ud, d, filter_regex)
return self._latest_versionstring_from_api(ud, d, filter_regex)
def _latest_versionstring_from_api(self, ud, d, filter_regex=None):
"""
Parse the latest version from a [name]/versions JSON API response.
"""
json_data = json.loads(self._fetch_index(ud.versionsurl, ud, d))
versions = [(0, i["num"], "") for i in json_data["versions"]]
if filter_regex:
versions = [v for v in versions if re.match(filter_regex, v[1])]
versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp))
return (versions[-1][1], "") if versions else ("", "")
def _latest_versionstring_from_index(self, ud, d, filter_regex=None):
"""
Parse the latest version from a Cargo sparse index file (NDJSON).
https://doc.rust-lang.org/cargo/reference/registry-index.html#index-files
"""
versions = []
response = self._fetch_index(ud.versionsurl, ud, d)
for line in response.splitlines():
data = json.loads(line)
if not data.get("yanked", False):
versions.append((0, data["vers"], ""))
if filter_regex:
versions = [v for v in versions if re.match(filter_regex, v[1])]
versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp))
return (versions[-1][1], "") if versions else ("", "")
|
