# Copyright (C) 2016-2018 Wind River Systems, Inc. # # SPDX-License-Identifier: GPL-2.0-only # import logging import json import os from urllib.parse import unquote from urllib.parse import urlparse import bb import layerindexlib import layerindexlib.plugin logger = logging.getLogger('BitBake.layerindexlib.restapi') def plugin_init(plugins): return RestApiPlugin() class RestApiPlugin(layerindexlib.plugin.IndexPlugin): def __init__(self): self.type = "restapi" def load_index(self, url, load): """ Fetches layer information from a local or remote layer index. The return value is a LayerIndexObj. url is the url to the rest api of the layer index, such as: https://layers.openembedded.org/layerindex/api/ Or a local file... """ up = urlparse(url) if up.scheme == 'file': return self.load_index_file(up, url, load) if up.scheme == 'http' or up.scheme == 'https': return self.load_index_web(up, url, load) raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url) def load_index_file(self, up, url, load): """ Fetches layer information from a local file or directory. The return value is a LayerIndexObj. ud is the parsed url to the local file or directory. """ if not os.path.exists(up.path): raise FileNotFoundError(up.path) index = layerindexlib.LayerIndexObj() index.config = {} index.config['TYPE'] = self.type index.config['URL'] = url params = self.layerindex._parse_params(up.params) if 'desc' in params: index.config['DESCRIPTION'] = unquote(params['desc']) else: index.config['DESCRIPTION'] = up.path if 'cache' in params: index.config['CACHE'] = params['cache'] if 'branch' in params: branches = params['branch'].split(',') index.config['BRANCH'] = branches else: branches = ['*'] def load_cache(path, index, branches=[]): logger.debug('Loading json file %s' % path) with open(path, 'rt', encoding='utf-8') as f: pindex = json.load(f) # Filter the branches on loaded files... newpBranch = [] for branch in branches: if branch != '*': if 'branches' in pindex: for br in pindex['branches']: if br['name'] == branch: newpBranch.append(br) else: if 'branches' in pindex: for br in pindex['branches']: newpBranch.append(br) if newpBranch: index.add_raw_element('branches', layerindexlib.Branch, newpBranch) else: logger.debug('No matching branches (%s) in index file(s)' % branches) # No matching branches.. return nothing... return for (lName, lType) in [("layerItems", layerindexlib.LayerItem), ("layerBranches", layerindexlib.LayerBranch), ("layerDependencies", layerindexlib.LayerDependency), ("recipes", layerindexlib.Recipe), ("machines", layerindexlib.Machine), ("distros", layerindexlib.Distro)]: if lName in pindex: index.add_raw_element(lName, lType, pindex[lName]) if not os.path.isdir(up.path): load_cache(up.path, index, branches) return index logger.debug('Loading from dir %s...' % (up.path)) for (dirpath, _, filenames) in os.walk(up.path): for filename in filenames: if not filename.endswith('.json'): continue fpath = os.path.join(dirpath, filename) load_cache(fpath, index, branches) return index def load_index_web(self, up, url, load): """ Fetches layer information from a remote layer index. The return value is a LayerIndexObj. ud is the parsed url to the rest api of the layer index, such as: https://layers.openembedded.org/layerindex/api/ """ def _get_json_response(apiurl=None, username=None, password=None, retry=True): assert apiurl is not None logger.debug("fetching %s" % apiurl) up = urlparse(apiurl) username=up.username password=up.password # Strip username/password and params if up.port: up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port)) else: up_stripped = up._replace(params="", netloc=up.hostname) res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password) try: parsed = json.loads(res.read().decode('utf-8')) except ConnectionResetError: if retry: logger.debug("%s: Connection reset by peer. Retrying..." % url) parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False) logger.debug("%s: retry successful.") else: raise layerindexlib.LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl) return parsed index = layerindexlib.LayerIndexObj() index.config = {} index.config['TYPE'] = self.type index.config['URL'] = url params = self.layerindex._parse_params(up.params) if 'desc' in params: index.config['DESCRIPTION'] = unquote(params['desc']) else: index.config['DESCRIPTION'] = up.hostname if 'cache' in params: index.config['CACHE'] = params['cache'] if 'branch' in params: branches = params['branch'].split(',') index.config['BRANCH'] = branches else: branches = ['*'] try: index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password) except Exception as e: raise layerindexlib.LayerIndexFetchError(url, e) # Local raw index set... pindex = {} # Load all the requested branches at the same time time, # a special branch of '*' means load all branches filter = "" if "*" not in branches: filter = "?filter=name:%s" % "OR".join(branches) logger.debug("Loading %s from %s" % (branches, index.apilinks['branches'])) # The link won't include username/password, so pull it from the original url pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter, username=up.username, password=up.password) if not pindex['branches']: logger.debug("No valid branches (%s) found at url %s." % (branch, url)) return index index.add_raw_element("branches", layerindexlib.Branch, pindex['branches']) # Load all of the layerItems (these can not be easily filtered) logger.debug("Loading %s from %s" % ('layerItems', index.apilinks['layerItems'])) # The link won't include username/password, so pull it from the original url pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'], username=up.username, password=up.password) if not pindex['layerItems']: logger.debug("No layers were found at url %s." % (url)) return index index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems']) # From this point on load the contents for each branch. Otherwise we # could run into a timeout. for branch in index.branches: filter = "?filter=branch__name:%s" % index.branches[branch].name logger.debug("Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches'])) # The link won't include username/password, so pull it from the original url pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter, username=up.username, password=up.password) if not pindex['layerBranches']: logger.debug("No valid layer branches (%s) found at url %s." % (branches or "*", url)) return index index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches']) # Load the rest, they all have a similar format # Note: the layer index has a few more items, we can add them if necessary # in the future. filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency), ("recipes", layerindexlib.Recipe), ("machines", layerindexlib.Machine), ("distros", layerindexlib.Distro)]: if lName not in load: continue logger.debug("Loading %s from %s" % (lName, index.apilinks[lName])) # The link won't include username/password, so pull it from the original url pindex[lName] = _get_json_response(index.apilinks[lName] + filter, username=up.username, password=up.password) index.add_raw_element(lName, lType, pindex[lName]) return index def store_index(self, url, index): """ Store layer information into a local file/dir. The return value is a dictionary containing API, layer, branch, dependency, recipe, machine, distro, information. ud is a parsed url to a directory or file. If the path is a directory, we will split the files into one file per layer. If the path is to a file (exists or not) the entire DB will be dumped into that one file. """ up = urlparse(url) if up.scheme != 'file': raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url) logger.debug("Storing to %s..." % up.path) try: layerbranches = index.layerBranches except KeyError: logger.error('No layerBranches to write.') return def filter_item(layerbranchid, objects): filtered = [] for obj in getattr(index, objects, None): try: if getattr(index, objects)[obj].layerbranch_id == layerbranchid: filtered.append(getattr(index, objects)[obj]._data) except AttributeError: logger.debug('No obj.layerbranch_id: %s' % objects) # No simple filter method, just include it... try: filtered.append(getattr(index, objects)[obj]._data) except AttributeError: logger.debug('No obj._data: %s %s' % (objects, type(obj))) filtered.append(obj) return filtered # Write out to a single file. # Filter out unnecessary items, then sort as we write for determinism if not os.path.isdir(up.path): pindex = {} pindex['branches'] = [] pindex['layerItems'] = [] pindex['layerBranches'] = [] for layerbranchid in layerbranches: if layerbranches[layerbranchid].branch._data not in pindex['branches']: pindex['branches'].append(layerbranches[layerbranchid].branch._data) if layerbranches[layerbranchid].layer._data not in pindex['layerItems']: pindex['layerItems'].append(layerbranches[layerbranchid].layer._data) if layerbranches[layerbranchid]._data not in pindex['layerBranches']: pindex['layerBranches'].append(layerbranches[layerbranchid]._data) for entry in index._index: # Skip local items, apilinks and items already processed if entry in index.config['local'] or \ entry == 'apilinks' or \ entry == 'branches' or \ entry == 'layerBranches' or \ entry == 'layerItems': continue if entry not in pindex: pindex[entry] = [] pindex[entry].extend(filter_item(layerbranchid, entry)) bb.debug(1, 'Writing index to %s' % up.path) with open(up.path, 'wt') as f: json.dump(layerindexlib.sort_entry(pindex), f, indent=4) return # Write out to a directory one file per layerBranch # Prepare all layer related items, to create a minimal file. # We have to sort the entries as we write so they are deterministic for layerbranchid in layerbranches: pindex = {} for entry in index._index: # Skip local items, apilinks and items already processed if entry in index.config['local'] or \ entry == 'apilinks' or \ entry == 'branches' or \ entry == 'layerBranches' or \ entry == 'layerItems': continue pindex[entry] = filter_item(layerbranchid, entry) # Add the layer we're processing as the first one... pindex['branches'] = [layerbranches[layerbranchid].branch._data] pindex['layerItems'] = [layerbranches[layerbranchid].layer._data] pindex['layerBranches'] = [layerbranches[layerbranchid]._data] # We also need to include the layerbranch for any dependencies... for layerdep in pindex['layerDependencies']: layerdependency = layerindexlib.LayerDependency(index, layerdep) layeritem = layerdependency.dependency layerbranch = layerdependency.dependency_layerBranch # We need to avoid duplicates... if layeritem._data not in pindex['layerItems']: pindex['layerItems'].append(layeritem._data) if layerbranch._data not in pindex['layerBranches']: pindex['layerBranches'].append(layerbranch._data) # apply mirroring adjustments here.... fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name'] fname = fname.translate(str.maketrans('/ ', '__')) fpath = os.path.join(up.path, fname) bb.debug(1, 'Writing index to %s' % fpath + '.json') with open(fpath + '.json', 'wt') as f: json.dump(layerindexlib.sort_entry(pindex), f, indent=4)