1
// Copyright (c) 2017-2022 Cloudflare, Inc.
2
// Licensed under the Apache 2.0 license found in the LICENSE file or at:
3
//     https://opensource.org/licenses/Apache-2.0
4
#include "requirements.h"
5
 
6
#include <capnp/compat/json.h>
7
#include <capnp/message.h>
8
#include <kj/debug.h>
9
 
10
#include <cctype>
11
 
12
namespace workerd::api::pyodide {
13
 
14
// getField gets a field of a JSON object by key
15
capnp::json::Value::Reader getField(
16
    capnp::List<::capnp::json::Value::Field, capnp::Kind::STRUCT>::Reader &object,
17
    kj::StringPtr name) {
18
  for (const auto &ent: object) {
19
    if (ent.getName() == name) {
20
      return ent.getValue();
21
    }
22
  }
23
 
24
  KJ_FAIL_ASSERT("Expected key in JSON object", name);
25
}
26
 
27
kj::String canonicalizePythonPackageName(kj::StringPtr name) {
28
  kj::Vector<char> res(name.size());
29
 
30
  auto isSeparator = [](char c) { return c == '-' || c == '_' || c == '.'; };
31
 
32
  for (int i = 0; i < name.size(); i++) {
33
    if (isSeparator(name[i])) {
34
      res.add('-');
35
      // make i point to the last separator in the sequence
36
      while (isSeparator(name[i])) i++;
37
      i--;
38
      continue;
39
    }
40
 
41
    res.add(std::tolower(name[i]));
42
  }
43
 
44
  res.add(0);  // NUL terminator
45
 
46
  return kj::String(res.releaseAsArray());
47
}
48
 
49
// getDepMapFromPackagesLock computes a dependency map (a mapping from requirement to list of dependencies) from the Pyodide lock file JSON
50
DepMap getDepMapFromPackagesLock(
51
    capnp::List<capnp::json::Value::Field, capnp::Kind::STRUCT>::Reader &packages) {
52
  DepMap res;
53
 
54
  for (const auto &ent: packages) {
55
    auto packageObj = ent.getValue().getObject();
56
    auto depends = getField(packageObj, "depends").getArray();
57
 
58
    auto &[_, deps] = res.insert(kj::str(ent.getName()), kj::Vector<kj::String>(depends.size()));
59
 
60
    for (const auto &dep: depends) {
61
      deps.add(kj::str(dep.getString()));
62
    }
63
  }
64
 
65
  return res;
66
}
67
 
68
// addWithRecursiveDependencies adds a requirement along with all its dependencies (according to the dependency map) to the requirements set
69
void addWithRecursiveDependencies(
70
    kj::StringPtr requirement, const DepMap &depMap, kj::HashSet<kj::String> &requirementsSet) {
71
  auto normalizedName = canonicalizePythonPackageName(requirement);
72
  if (requirementsSet.contains(normalizedName)) {
73
    return;
74
  }
75
 
76
  requirementsSet.insert(kj::str(normalizedName));
77
 
78
  KJ_IF_SOME(deps, depMap.find(normalizedName)) {
79
    for (const auto &dep: deps) {
80
      addWithRecursiveDependencies(dep, depMap, requirementsSet);
81
    }
82
  }
83
}
84
 
85
kj::Own<capnp::List<capnp::json::Value::Field>::Reader> parseLockFile(
86
    kj::StringPtr lockFileContents) {
87
  capnp::JsonCodec json;
88
  capnp::MallocMessageBuilder message;
89
 
90
  auto lock = message.initRoot<capnp::JsonValue>();
91
  json.decodeRaw(lockFileContents, lock);
92
 
93
  auto object = lock.getObject().asReader();
94
  auto packages = getField(object, "packages").getObject();
95
  return capnp::clone(packages);
96
}
97
 
98
kj::HashSet<kj::String> getPythonPackageNames(
99
    capnp::List<capnp::json::Value::Field>::Reader packages,
100
    const DepMap &depMap,
101
    kj::ArrayPtr<kj::String> requirements,
102
    kj::StringPtr packagesVersion) {
103
 
104
  kj::HashSet<kj::String> allRequirements;  // Requirements including their recursive dependencies.
105
 
106
  // Potentially add the stdlib packages and their recursive dependencies.
107
  // TODO: Loading stdlib and its dependencies breaks package snapshots on "20240829.4".
108
  // Remove this version check once a new package/python release is made.
109
  if (packagesVersion != "20240829.4") {
110
    // We need to scan the packages list for any packages that need to be included because they
111
    // are part of Python's stdlib (hashlib etc). These need to be implicitly treated as part of
112
    // our `requirements`.
113
    for (const auto &ent: packages) {
114
      auto name = ent.getName();
115
      auto obj = ent.getValue().getObject();
116
      auto packageType = getField(obj, "package_type").getString();
117
 
118
      if (packageType == "cpython_module"_kj) {
119
        addWithRecursiveDependencies(name, depMap, allRequirements);
120
      }
121
    }
122
  }
123
 
124
  // Add all recursive dependencies of each requirement.
125
  for (const auto &req: requirements) {
126
    addWithRecursiveDependencies(req, depMap, allRequirements);
127
  }
128
 
129
  return allRequirements;
130
}
131
 
132
}  // namespace workerd::api::pyodide