vibefpmgui.manifest

  1"""Reading fpm manifests and resolving the dependency tree (no Qt in here).
  2
  3The tree is built by starting at the main ``fpm.toml`` and following every
  4dependency to its own ``fpm.toml``.  Where a dependency lives on disk is taken
  5from (in order of preference):
  6
  71. ``path = "..."`` in the dependency spec (relative to the declaring package),
  82. ``build/cache.toml`` written by fpm after it resolves the dependencies,
  93. the default fpm download location ``build/dependencies/<name>``.
 10
 11Dependencies that have not been downloaded yet are still shown, but flagged as
 12missing (run ``fpm update`` or ``fpm build`` to fetch them).
 13"""
 14
 15from __future__ import annotations
 16
 17import re
 18import tomllib
 19from dataclasses import dataclass, field
 20from pathlib import Path
 21
 22# Names that fpm treats as metapackages when given as ``name = "*"``.
 23METAPACKAGES = {
 24    "openmp", "stdlib", "minpack", "mpi", "hdf5", "netcdf", "blas", "lapack",
 25}
 26
 27# Node kinds
 28ROOT = "root"
 29GIT = "git"
 30PATH = "path"
 31REGISTRY = "registry"
 32META = "metapackage"
 33MISSING = "missing"
 34DEV_ONLY = "dev-only"  # a dependency's dev-dependency: fpm doesn't download it (see resolve)
 35
 36
 37@dataclass
 38class Package:
 39    """A node in the dependency graph."""
 40
 41    name: str
 42    kind: str
 43    version: str = ""
 44    manifest_version: str = ""  # the version its own fpm.toml gives ("": none)
 45    directory: Path | None = None
 46    manifest: Path | None = None
 47    license: str = ""
 48    author: str = ""
 49    description: str = ""
 50    homepage: str = ""
 51    git: str = ""
 52    git_ref: str = ""
 53    git_rev: str = ""
 54    error: str = ""
 55    asked_kind: str = ""  # a DEV_ONLY package's: the kind it would be (GIT, PATH, REGISTRY or META)
 56
 57
 58@dataclass
 59class Edge:
 60    parent: str
 61    child: str
 62    dev: bool = False  # from [dev-dependencies] / test or example targets
 63
 64
 65@dataclass
 66class Declaration:
 67    """One place a dependency is declared: which package asks for it, and how."""
 68    parent: str
 69    spec: object       # as written in the parent's fpm.toml
 70    dev: bool = False
 71    parent_dir: Path | None = None  # (for resolving a relative ``path``)
 72
 73
 74@dataclass
 75class DependencyGraph:
 76    root: str
 77    packages: dict[str, Package] = field(default_factory=dict)
 78    edges: list[Edge] = field(default_factory=list)
 79    # Every declaration of each dependency, closest to the root first (the
 80    # first is the one fpm uses).
 81    declarations: dict[str, list[Declaration]] = field(default_factory=dict)
 82
 83    def children(self, name: str) -> list[Edge]:
 84        return [e for e in self.edges if e.parent == name]
 85
 86    def parents(self, name: str) -> list[Edge]:
 87        return [e for e in self.edges if e.child == name]
 88
 89
 90def load_toml(path: Path) -> dict:
 91    with open(path, "rb") as f:
 92        return tomllib.load(f)
 93
 94
 95def _read_version(value, package_dir: Path) -> str:
 96    """fpm allows ``version`` to be a string or the name of a file holding it."""
 97    if not isinstance(value, str):
 98        return ""
 99    candidate = package_dir / value
100    if candidate.is_file():
101        try:
102            return candidate.read_text().strip().splitlines()[0].strip()
103        except (OSError, IndexError):
104            return value
105    return value
106
107
108def _as_text(value) -> str:
109    if isinstance(value, list):
110        return ", ".join(str(v) for v in value)
111    return "" if value is None else str(value)
112
113
114def _collect_deps(data: dict, is_root: bool) -> list[tuple[str, object, bool]]:
115    """Return ``(name, spec, dev)`` for every dependency a manifest declares.
116
117    Dev-dependencies and per-target dependencies are only used by fpm for the
118    root project (``is_root``); for a downloaded package they're left out
119    (``is_root`` False), though `resolve` can ask for them to show them.
120    """
121    deps: dict[str, tuple[object, bool]] = {}
122
123    def add(table, dev):
124        if not isinstance(table, dict):
125            return
126        for name, spec in table.items():
127            # A normal dependency wins over a dev one of the same name.
128            if name not in deps or (deps[name][1] and not dev):
129                deps[name] = (spec, dev)
130
131    add(data.get("dependencies"), False)
132    if is_root:
133        add(data.get("dev-dependencies"), True)
134        for target in data.get("executable", []) or []:
135            add(target.get("dependencies"), False)
136        for section in ("test", "example"):
137            for target in data.get(section, []) or []:
138                add(target.get("dependencies"), True)
139    return [(name, spec, dev) for name, (spec, dev) in deps.items()]
140
141
142def _fill_from_manifest(pkg: Package, data: dict) -> None:
143    if pkg.directory is not None:
144        pkg.manifest_version = _read_version(data.get("version"), pkg.directory)
145        pkg.version = pkg.manifest_version or pkg.version
146    pkg.license = _as_text(data.get("license"))
147    pkg.author = _as_text(data.get("author"))
148    pkg.description = _as_text(data.get("description"))
149    pkg.homepage = _as_text(data.get("homepage"))
150
151
152def resolve(manifest_path: str | Path, dependencies_dev: bool = False) -> DependencyGraph:
153    """Build the dependency graph for the project whose manifest is given.
154
155    ``dependencies_dev``: also the dependencies' own dev-dependencies (and
156    their test and example targets' dependencies), as dev edges. fpm uses
157    only the project's own, so it never downloads these: one that isn't
158    also a dependency fpm does use is a `DEV_ONLY` package (no folder or
159    manifest), and their declarations aren't the graph's (`conflicts` is
160    about what fpm uses).
161
162    Raises ``OSError``/``tomllib.TOMLDecodeError`` if the main manifest can't
163    be read; problems with dependency manifests are recorded on the nodes.
164    """
165    manifest_path = Path(manifest_path).resolve()
166    root_dir = manifest_path.parent
167    data = load_toml(manifest_path)
168
169    root_name = str(data.get("name", root_dir.name))
170    graph = DependencyGraph(root=root_name)
171    root = Package(root_name, ROOT, directory=root_dir, manifest=manifest_path)
172    _fill_from_manifest(root, data)
173    graph.packages[root_name] = root
174
175    cache: dict = {}
176    cache_file = root_dir / "build" / "cache.toml"
177    if cache_file.is_file():
178        try:
179            cache = load_toml(cache_file)
180        except (OSError, tomllib.TOMLDecodeError):
181            cache = {}
182
183    # Breadth-first so the first spec seen for a package (closest to the
184    # root) is the one used, which matches how fpm resolves duplicates.
185    queue: list[tuple[Package, dict]] = [(root, data)]
186    unused: list[tuple[str, str, object]] = []  # (parent, name, spec) of the dependencies' dev-dependencies
187    while queue:
188        parent, parent_data = queue.pop(0)
189        is_root = parent is root
190        if dependencies_dev and not is_root:
191            used = {name for name, _spec, _dev in _collect_deps(parent_data, False)}
192            unused += [(parent.name, name, spec) for name, spec, _dev in _collect_deps(parent_data, True)
193                       if name not in used]
194        for name, spec, dev in _collect_deps(parent_data, is_root):
195            graph.edges.append(Edge(parent.name, name, dev))
196            graph.declarations.setdefault(name, []).append(
197                Declaration(parent.name, spec, dev, parent.directory or root_dir))
198            if name in graph.packages:
199                continue
200            pkg, pkg_data = _locate(name, spec, parent, root_dir, cache.get(name, {}))
201            graph.packages[name] = pkg
202            if pkg_data is not None:
203                queue.append((pkg, pkg_data))
204    # After every dependency fpm uses is in: a dev-dependency that's also one
205    # of those is that package (downloaded); only the others are DEV_ONLY.
206    for parent_name, name, spec in unused:
207        graph.edges.append(Edge(parent_name, name, True))
208        if name not in graph.packages:
209            graph.packages[name] = _dev_only(name, spec, parent_name)
210    return graph
211
212
213def _dev_only(name: str, spec, parent: str) -> Package:
214    """A dependency's dev-dependency fpm doesn't download: what its manifest
215    asks for (no folder, no manifest)."""
216    pkg = Package(name, DEV_ONLY, error=f"A dev-dependency of {parent}: fpm doesn't download a dependency's "
217                                        "dev-dependencies (only the project's own).")
218    if isinstance(spec, str):
219        pkg.version = "" if spec == "*" else spec
220        pkg.asked_kind = META if name in METAPACKAGES else REGISTRY
221    elif isinstance(spec, dict):
222        pkg.asked_kind = PATH if "path" in spec else GIT if "git" in spec else REGISTRY
223        pkg.git = str(spec.get("git", ""))
224        for key in ("tag", "branch", "rev"):
225            if key in spec:
226                pkg.git_ref = f"{key} {spec[key]}"
227                break
228        pkg.version = str(spec.get("v", ""))
229    return pkg
230
231
232def _locate(name, spec, parent: Package, root_dir: Path, cached: dict):
233    """Create the Package for a dependency and load its manifest if present."""
234    if isinstance(spec, str):
235        # ``openmp = "*"`` style metapackage (or a bare registry version).
236        kind = META if name in METAPACKAGES else REGISTRY
237        return Package(name, kind, version="" if spec == "*" else spec), None
238    if not isinstance(spec, dict):
239        return Package(name, MISSING, error=f"Unrecognised dependency spec: {spec!r}"), None
240
241    pkg = Package(name, GIT)
242    directory: Path | None = None
243    if "path" in spec:
244        pkg.kind = PATH
245        directory = ((parent.directory or root_dir) / str(spec["path"])).resolve()
246    else:
247        if "git" in spec:
248            pkg.git = str(spec["git"])
249            for key in ("tag", "branch", "rev"):
250                if key in spec:
251                    pkg.git_ref = f"{key} {spec[key]}"
252                    break
253        else:
254            pkg.kind = REGISTRY
255            pkg.version = str(spec.get("v", ""))
256        if cached.get("proj-dir"):
257            directory = (root_dir / str(cached["proj-dir"])).resolve()
258        else:
259            directory = root_dir / "build" / "dependencies" / name
260        pkg.git = pkg.git or str(cached.get("git", ""))
261        # The commit actually checked out; build/cache.toml's "rev" can be
262        # stale (fpm update leaves the old one there).
263        pkg.git_rev = checked_out_commit(directory) or str(cached.get("rev", ""))
264    # (fpm writes "0" for a package whose manifest gives no version: none, then.)
265    if cached.get("version") and str(cached["version"]) != "0" and not pkg.version:
266        pkg.version = str(cached["version"])
267
268    manifest = directory / "fpm.toml"
269    if not manifest.is_file():
270        pkg.directory = directory
271        if pkg.kind == PATH:
272            pkg.error = f"No fpm.toml found in {directory}"
273        else:
274            pkg.error = "Not downloaded yet (run fpm update or fpm build)."
275        pkg.kind = MISSING
276        return pkg, None
277
278    pkg.directory = directory
279    pkg.manifest = manifest
280    try:
281        data = load_toml(manifest)
282    except (OSError, tomllib.TOMLDecodeError) as exc:
283        pkg.error = f"Could not read {manifest}: {exc}"
284        return pkg, None
285    _fill_from_manifest(pkg, data)
286    return pkg, data
287
288
289def checked_out_commit(directory: Path) -> str:
290    """The commit a git checkout is at, read from .git/HEAD (no git needed);
291    "" if it isn't a git checkout."""
292    git = directory / ".git"
293    try:
294        head = (git / "HEAD").read_text().strip()
295    except OSError:
296        return ""
297    if not head.startswith("ref:"):
298        return head if re.fullmatch(r"[0-9a-f]{40,64}", head) else ""
299    ref = head[4:].strip()
300    try:
301        return (git / ref).read_text().strip()
302    except OSError:
303        pass
304    try:  # packed refs: "<sha> <ref>" lines
305        for line in (git / "packed-refs").read_text().splitlines():
306            sha, _, name = line.partition(" ")
307            if name.strip() == ref:
308                return sha
309    except OSError:
310        pass
311    return ""
312
313
314def project_features(data: dict) -> dict[str, list[str]]:
315    """Features declared in a manifest's ``[features]`` table, each with the
316    (dotted) properties it sets, e.g. ``{"mpi": ["dependencies.mpi", "flags"]}``."""
317    table = data.get("features")
318    if not isinstance(table, dict):
319        return {}
320
321    def paths(value, prefix=""):
322        if isinstance(value, dict):
323            out = []
324            for key, sub in value.items():
325                out += paths(sub, f"{prefix}.{key}" if prefix else str(key))
326            return out
327        return [prefix]
328
329    return {str(name): paths(value) for name, value in table.items()}
330
331
332def project_profiles(data: dict) -> dict[str, list[str]]:
333    """Feature profiles declared in ``[profiles]`` (name -> list of features)."""
334    table = data.get("profiles")
335    if not isinstance(table, dict):
336        return {}
337    return {str(name): [str(f) for f in value] if isinstance(value, list) else []
338            for name, value in table.items()}
339
340
341# ------------------------------------------------------------ conflicts
342def _git_url(url: str) -> str:
343    """A git URL in a form where the same repository compares equal."""
344    url = url.strip().rstrip("/").lower()
345    url = re.sub(r"\.git$", "", url)
346    url = re.sub(r"^git@([^:]+):", r"https://\1/", url)   # git@github.com:a/b -> https://github.com/a/b
347    return re.sub(r"^(https?|git|ssh)://(www\.)?", "", url)
348
349
350def spec_key(declaration: Declaration) -> tuple:
351    """What a declaration asks for, so that different ways of writing the
352    same thing compare equal."""
353    spec = declaration.spec
354    if isinstance(spec, str):
355        return ("version", spec.strip())
356    if not isinstance(spec, dict):
357        return ("other", repr(spec))
358    if "path" in spec:
359        return ("path", str(((declaration.parent_dir or Path()) / str(spec["path"])).resolve()))
360    if "git" in spec:
361        for key in ("tag", "branch", "rev"):
362            if key in spec:
363                value = str(spec[key]).strip()
364                # A commit may be written short or long: compare 7 characters.
365                return ("git", _git_url(str(spec["git"])), key, value[:7] if key == "rev" else value)
366        return ("git", _git_url(str(spec["git"])), "", "")
367    return ("registry", *sorted((str(k), str(v)) for k, v in spec.items()))
368
369
370def describe_spec(spec) -> str:
371    """A declaration in a few words: "tag v0.5.0", "path ../lib", "any version"…"""
372    if isinstance(spec, str):
373        return "any version" if spec.strip() == "*" else f"version {spec}"
374    if not isinstance(spec, dict):
375        return repr(spec)
376    if "path" in spec:
377        return f"path {spec['path']}"
378    if "git" in spec:
379        for key in ("tag", "branch", "rev"):
380            if key in spec:
381                value = str(spec[key])
382                return f"{key} {value[:7] if key == 'rev' else value} of {spec['git']}"
383        return f"default branch of {spec['git']}"
384    if "v" in spec:
385        return f"version {spec['v']}"
386    return ", ".join(f"{k} {v}" for k, v in spec.items())
387
388
389def conflicts(graph: DependencyGraph) -> dict[str, list[Declaration]]:
390    """Dependencies that different packages ask for differently (another
391    tag, commit, branch, repository or path). fpm uses only the declaration
392    closest to the project, silently: these are worth knowing about."""
393    out = {}
394    for name, declarations in graph.declarations.items():
395        if len({spec_key(d) for d in declarations}) > 1:
396            out[name] = declarations
397    return out