node-env.nix 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574
  1. # This file originates from node2nix
  2. {lib, stdenv, nodejs, python2, pkgs, libtool, runCommand, writeTextFile}:
  3. let
  4. # Workaround to cope with utillinux in Nixpkgs 20.09 and util-linux in Nixpkgs master
  5. utillinux = if pkgs ? utillinux then pkgs.utillinux else pkgs.util-linux;
  6. python = if nodejs ? python then nodejs.python else python2;
  7. # Create a tar wrapper that filters all the 'Ignoring unknown extended header keyword' noise
  8. tarWrapper = runCommand "tarWrapper" {} ''
  9. mkdir -p $out/bin
  10. cat > $out/bin/tar <<EOF
  11. #! ${stdenv.shell} -e
  12. $(type -p tar) "\$@" --warning=no-unknown-keyword --delay-directory-restore
  13. EOF
  14. chmod +x $out/bin/tar
  15. '';
  16. # Function that generates a TGZ file from a NPM project
  17. buildNodeSourceDist =
  18. { name, version, src, ... }:
  19. stdenv.mkDerivation {
  20. name = "node-tarball-${name}-${version}";
  21. inherit src;
  22. buildInputs = [ nodejs ];
  23. buildPhase = ''
  24. export HOME=$TMPDIR
  25. tgzFile=$(npm pack | tail -n 1) # Hooks to the pack command will add output (https://docs.npmjs.com/misc/scripts)
  26. '';
  27. installPhase = ''
  28. mkdir -p $out/tarballs
  29. mv $tgzFile $out/tarballs
  30. mkdir -p $out/nix-support
  31. echo "file source-dist $out/tarballs/$tgzFile" >> $out/nix-support/hydra-build-products
  32. '';
  33. };
  34. includeDependencies = {dependencies}:
  35. lib.optionalString (dependencies != [])
  36. (lib.concatMapStrings (dependency:
  37. ''
  38. # Bundle the dependencies of the package
  39. mkdir -p node_modules
  40. cd node_modules
  41. # Only include dependencies if they don't exist. They may also be bundled in the package.
  42. if [ ! -e "${dependency.name}" ]
  43. then
  44. ${composePackage dependency}
  45. fi
  46. cd ..
  47. ''
  48. ) dependencies);
  49. # Recursively composes the dependencies of a package
  50. composePackage = { name, packageName, src, dependencies ? [], ... }@args:
  51. builtins.addErrorContext "while evaluating node package '${packageName}'" ''
  52. DIR=$(pwd)
  53. cd $TMPDIR
  54. unpackFile ${src}
  55. # Make the base dir in which the target dependency resides first
  56. mkdir -p "$(dirname "$DIR/${packageName}")"
  57. if [ -f "${src}" ]
  58. then
  59. # Figure out what directory has been unpacked
  60. packageDir="$(find . -maxdepth 1 -type d | tail -1)"
  61. # Restore write permissions to make building work
  62. find "$packageDir" -type d -exec chmod u+x {} \;
  63. chmod -R u+w "$packageDir"
  64. # Move the extracted tarball into the output folder
  65. mv "$packageDir" "$DIR/${packageName}"
  66. elif [ -d "${src}" ]
  67. then
  68. # Get a stripped name (without hash) of the source directory.
  69. # On old nixpkgs it's already set internally.
  70. if [ -z "$strippedName" ]
  71. then
  72. strippedName="$(stripHash ${src})"
  73. fi
  74. # Restore write permissions to make building work
  75. chmod -R u+w "$strippedName"
  76. # Move the extracted directory into the output folder
  77. mv "$strippedName" "$DIR/${packageName}"
  78. fi
  79. # Unset the stripped name to not confuse the next unpack step
  80. unset strippedName
  81. # Include the dependencies of the package
  82. cd "$DIR/${packageName}"
  83. ${includeDependencies { inherit dependencies; }}
  84. cd ..
  85. ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
  86. '';
  87. pinpointDependencies = {dependencies, production}:
  88. let
  89. pinpointDependenciesFromPackageJSON = writeTextFile {
  90. name = "pinpointDependencies.js";
  91. text = ''
  92. var fs = require('fs');
  93. var path = require('path');
  94. function resolveDependencyVersion(location, name) {
  95. if(location == process.env['NIX_STORE']) {
  96. return null;
  97. } else {
  98. var dependencyPackageJSON = path.join(location, "node_modules", name, "package.json");
  99. if(fs.existsSync(dependencyPackageJSON)) {
  100. var dependencyPackageObj = JSON.parse(fs.readFileSync(dependencyPackageJSON));
  101. if(dependencyPackageObj.name == name) {
  102. return dependencyPackageObj.version;
  103. }
  104. } else {
  105. return resolveDependencyVersion(path.resolve(location, ".."), name);
  106. }
  107. }
  108. }
  109. function replaceDependencies(dependencies) {
  110. if(typeof dependencies == "object" && dependencies !== null) {
  111. for(var dependency in dependencies) {
  112. var resolvedVersion = resolveDependencyVersion(process.cwd(), dependency);
  113. if(resolvedVersion === null) {
  114. process.stderr.write("WARNING: cannot pinpoint dependency: "+dependency+", context: "+process.cwd()+"\n");
  115. } else {
  116. dependencies[dependency] = resolvedVersion;
  117. }
  118. }
  119. }
  120. }
  121. /* Read the package.json configuration */
  122. var packageObj = JSON.parse(fs.readFileSync('./package.json'));
  123. /* Pinpoint all dependencies */
  124. replaceDependencies(packageObj.dependencies);
  125. if(process.argv[2] == "development") {
  126. replaceDependencies(packageObj.devDependencies);
  127. }
  128. replaceDependencies(packageObj.optionalDependencies);
  129. /* Write the fixed package.json file */
  130. fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2));
  131. '';
  132. };
  133. in
  134. ''
  135. node ${pinpointDependenciesFromPackageJSON} ${if production then "production" else "development"}
  136. ${lib.optionalString (dependencies != [])
  137. ''
  138. if [ -d node_modules ]
  139. then
  140. cd node_modules
  141. ${lib.concatMapStrings (dependency: pinpointDependenciesOfPackage dependency) dependencies}
  142. cd ..
  143. fi
  144. ''}
  145. '';
  146. # Recursively traverses all dependencies of a package and pinpoints all
  147. # dependencies in the package.json file to the versions that are actually
  148. # being used.
  149. pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }@args:
  150. ''
  151. if [ -d "${packageName}" ]
  152. then
  153. cd "${packageName}"
  154. ${pinpointDependencies { inherit dependencies production; }}
  155. cd ..
  156. ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
  157. fi
  158. '';
  159. # Extract the Node.js source code which is used to compile packages with
  160. # native bindings
  161. nodeSources = runCommand "node-sources" {} ''
  162. tar --no-same-owner --no-same-permissions -xf ${nodejs.src}
  163. mv node-* $out
  164. '';
  165. # Script that adds _integrity fields to all package.json files to prevent NPM from consulting the cache (that is empty)
  166. addIntegrityFieldsScript = writeTextFile {
  167. name = "addintegrityfields.js";
  168. text = ''
  169. var fs = require('fs');
  170. var path = require('path');
  171. function augmentDependencies(baseDir, dependencies) {
  172. for(var dependencyName in dependencies) {
  173. var dependency = dependencies[dependencyName];
  174. // Open package.json and augment metadata fields
  175. var packageJSONDir = path.join(baseDir, "node_modules", dependencyName);
  176. var packageJSONPath = path.join(packageJSONDir, "package.json");
  177. if(fs.existsSync(packageJSONPath)) { // Only augment packages that exist. Sometimes we may have production installs in which development dependencies can be ignored
  178. console.log("Adding metadata fields to: "+packageJSONPath);
  179. var packageObj = JSON.parse(fs.readFileSync(packageJSONPath));
  180. if(dependency.integrity) {
  181. packageObj["_integrity"] = dependency.integrity;
  182. } else {
  183. packageObj["_integrity"] = "sha1-000000000000000000000000000="; // When no _integrity string has been provided (e.g. by Git dependencies), add a dummy one. It does not seem to harm and it bypasses downloads.
  184. }
  185. if(dependency.resolved) {
  186. packageObj["_resolved"] = dependency.resolved; // Adopt the resolved property if one has been provided
  187. } else {
  188. packageObj["_resolved"] = dependency.version; // Set the resolved version to the version identifier. This prevents NPM from cloning Git repositories.
  189. }
  190. if(dependency.from !== undefined) { // Adopt from property if one has been provided
  191. packageObj["_from"] = dependency.from;
  192. }
  193. fs.writeFileSync(packageJSONPath, JSON.stringify(packageObj, null, 2));
  194. }
  195. // Augment transitive dependencies
  196. if(dependency.dependencies !== undefined) {
  197. augmentDependencies(packageJSONDir, dependency.dependencies);
  198. }
  199. }
  200. }
  201. if(fs.existsSync("./package-lock.json")) {
  202. var packageLock = JSON.parse(fs.readFileSync("./package-lock.json"));
  203. if(![1, 2].includes(packageLock.lockfileVersion)) {
  204. process.stderr.write("Sorry, I only understand lock file versions 1 and 2!\n");
  205. process.exit(1);
  206. }
  207. if(packageLock.dependencies !== undefined) {
  208. augmentDependencies(".", packageLock.dependencies);
  209. }
  210. }
  211. '';
  212. };
  213. # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes
  214. reconstructPackageLock = writeTextFile {
  215. name = "addintegrityfields.js";
  216. text = ''
  217. var fs = require('fs');
  218. var path = require('path');
  219. var packageObj = JSON.parse(fs.readFileSync("package.json"));
  220. var lockObj = {
  221. name: packageObj.name,
  222. version: packageObj.version,
  223. lockfileVersion: 1,
  224. requires: true,
  225. dependencies: {}
  226. };
  227. function augmentPackageJSON(filePath, dependencies) {
  228. var packageJSON = path.join(filePath, "package.json");
  229. if(fs.existsSync(packageJSON)) {
  230. var packageObj = JSON.parse(fs.readFileSync(packageJSON));
  231. dependencies[packageObj.name] = {
  232. version: packageObj.version,
  233. integrity: "sha1-000000000000000000000000000=",
  234. dependencies: {}
  235. };
  236. processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies);
  237. }
  238. }
  239. function processDependencies(dir, dependencies) {
  240. if(fs.existsSync(dir)) {
  241. var files = fs.readdirSync(dir);
  242. files.forEach(function(entry) {
  243. var filePath = path.join(dir, entry);
  244. var stats = fs.statSync(filePath);
  245. if(stats.isDirectory()) {
  246. if(entry.substr(0, 1) == "@") {
  247. // When we encounter a namespace folder, augment all packages belonging to the scope
  248. var pkgFiles = fs.readdirSync(filePath);
  249. pkgFiles.forEach(function(entry) {
  250. if(stats.isDirectory()) {
  251. var pkgFilePath = path.join(filePath, entry);
  252. augmentPackageJSON(pkgFilePath, dependencies);
  253. }
  254. });
  255. } else {
  256. augmentPackageJSON(filePath, dependencies);
  257. }
  258. }
  259. });
  260. }
  261. }
  262. processDependencies("node_modules", lockObj.dependencies);
  263. fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2));
  264. '';
  265. };
  266. prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}:
  267. let
  268. forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com";
  269. in
  270. ''
  271. # Pinpoint the versions of all dependencies to the ones that are actually being used
  272. echo "pinpointing versions of dependencies..."
  273. source $pinpointDependenciesScriptPath
  274. # Patch the shebangs of the bundled modules to prevent them from
  275. # calling executables outside the Nix store as much as possible
  276. patchShebangs .
  277. # Deploy the Node.js package by running npm install. Since the
  278. # dependencies have been provided already by ourselves, it should not
  279. # attempt to install them again, which is good, because we want to make
  280. # it Nix's responsibility. If it needs to install any dependencies
  281. # anyway (e.g. because the dependency parameters are
  282. # incomplete/incorrect), it fails.
  283. #
  284. # The other responsibilities of NPM are kept -- version checks, build
  285. # steps, postprocessing etc.
  286. export HOME=$TMPDIR
  287. cd "${packageName}"
  288. runHook preRebuild
  289. ${lib.optionalString bypassCache ''
  290. ${lib.optionalString reconstructLock ''
  291. if [ -f package-lock.json ]
  292. then
  293. echo "WARNING: Reconstruct lock option enabled, but a lock file already exists!"
  294. echo "This will most likely result in version mismatches! We will remove the lock file and regenerate it!"
  295. rm package-lock.json
  296. else
  297. echo "No package-lock.json file found, reconstructing..."
  298. fi
  299. node ${reconstructPackageLock}
  300. ''}
  301. node ${addIntegrityFieldsScript}
  302. ''}
  303. npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild
  304. if [ "''${dontNpmInstall-}" != "1" ]
  305. then
  306. # NPM tries to download packages even when they already exist if npm-shrinkwrap is used.
  307. rm -f npm-shrinkwrap.json
  308. npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} install
  309. fi
  310. '';
  311. # Builds and composes an NPM package including all its dependencies
  312. buildNodePackage =
  313. { name
  314. , packageName
  315. , version
  316. , dependencies ? []
  317. , buildInputs ? []
  318. , production ? true
  319. , npmFlags ? ""
  320. , dontNpmInstall ? false
  321. , bypassCache ? false
  322. , reconstructLock ? false
  323. , preRebuild ? ""
  324. , dontStrip ? true
  325. , unpackPhase ? "true"
  326. , buildPhase ? "true"
  327. , meta ? {}
  328. , ... }@args:
  329. let
  330. extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "preRebuild" "unpackPhase" "buildPhase" "meta" ];
  331. in
  332. stdenv.mkDerivation ({
  333. name = "${name}-${version}";
  334. buildInputs = [ tarWrapper python nodejs ]
  335. ++ lib.optional (stdenv.isLinux) utillinux
  336. ++ lib.optional (stdenv.isDarwin) libtool
  337. ++ buildInputs;
  338. inherit nodejs;
  339. inherit dontStrip; # Stripping may fail a build for some package deployments
  340. inherit dontNpmInstall preRebuild unpackPhase buildPhase;
  341. compositionScript = composePackage args;
  342. pinpointDependenciesScript = pinpointDependenciesOfPackage args;
  343. passAsFile = [ "compositionScript" "pinpointDependenciesScript" ];
  344. installPhase = ''
  345. # Create and enter a root node_modules/ folder
  346. mkdir -p $out/lib/node_modules
  347. cd $out/lib/node_modules
  348. # Compose the package and all its dependencies
  349. source $compositionScriptPath
  350. ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }}
  351. # Create symlink to the deployed executable folder, if applicable
  352. if [ -d "$out/lib/node_modules/.bin" ]
  353. then
  354. ln -s $out/lib/node_modules/.bin $out/bin
  355. fi
  356. # Create symlinks to the deployed manual page folders, if applicable
  357. if [ -d "$out/lib/node_modules/${packageName}/man" ]
  358. then
  359. mkdir -p $out/share
  360. for dir in "$out/lib/node_modules/${packageName}/man/"*
  361. do
  362. mkdir -p $out/share/man/$(basename "$dir")
  363. for page in "$dir"/*
  364. do
  365. ln -s $page $out/share/man/$(basename "$dir")
  366. done
  367. done
  368. fi
  369. # Run post install hook, if provided
  370. runHook postInstall
  371. '';
  372. meta = {
  373. # default to Node.js' platforms
  374. platforms = nodejs.meta.platforms;
  375. } // meta;
  376. } // extraArgs);
  377. # Builds a node environment (a node_modules folder and a set of binaries)
  378. buildNodeDependencies =
  379. { name
  380. , packageName
  381. , version
  382. , src
  383. , dependencies ? []
  384. , buildInputs ? []
  385. , production ? true
  386. , npmFlags ? ""
  387. , dontNpmInstall ? false
  388. , bypassCache ? false
  389. , reconstructLock ? false
  390. , dontStrip ? true
  391. , unpackPhase ? "true"
  392. , buildPhase ? "true"
  393. , ... }@args:
  394. let
  395. extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" ];
  396. in
  397. stdenv.mkDerivation ({
  398. name = "node-dependencies-${name}-${version}";
  399. buildInputs = [ tarWrapper python nodejs ]
  400. ++ lib.optional (stdenv.isLinux) utillinux
  401. ++ lib.optional (stdenv.isDarwin) libtool
  402. ++ buildInputs;
  403. inherit dontStrip; # Stripping may fail a build for some package deployments
  404. inherit dontNpmInstall unpackPhase buildPhase;
  405. includeScript = includeDependencies { inherit dependencies; };
  406. pinpointDependenciesScript = pinpointDependenciesOfPackage args;
  407. passAsFile = [ "includeScript" "pinpointDependenciesScript" ];
  408. installPhase = ''
  409. mkdir -p $out/${packageName}
  410. cd $out/${packageName}
  411. source $includeScriptPath
  412. # Create fake package.json to make the npm commands work properly
  413. cp ${src}/package.json .
  414. chmod 644 package.json
  415. ${lib.optionalString bypassCache ''
  416. if [ -f ${src}/package-lock.json ]
  417. then
  418. cp ${src}/package-lock.json .
  419. fi
  420. ''}
  421. # Go to the parent folder to make sure that all packages are pinpointed
  422. cd ..
  423. ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
  424. ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }}
  425. # Expose the executables that were installed
  426. cd ..
  427. ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
  428. mv ${packageName} lib
  429. ln -s $out/lib/node_modules/.bin $out/bin
  430. '';
  431. } // extraArgs);
  432. # Builds a development shell
  433. buildNodeShell =
  434. { name
  435. , packageName
  436. , version
  437. , src
  438. , dependencies ? []
  439. , buildInputs ? []
  440. , production ? true
  441. , npmFlags ? ""
  442. , dontNpmInstall ? false
  443. , bypassCache ? false
  444. , reconstructLock ? false
  445. , dontStrip ? true
  446. , unpackPhase ? "true"
  447. , buildPhase ? "true"
  448. , ... }@args:
  449. let
  450. nodeDependencies = buildNodeDependencies args;
  451. in
  452. stdenv.mkDerivation {
  453. name = "node-shell-${name}-${version}";
  454. buildInputs = [ python nodejs ] ++ lib.optional (stdenv.isLinux) utillinux ++ buildInputs;
  455. buildCommand = ''
  456. mkdir -p $out/bin
  457. cat > $out/bin/shell <<EOF
  458. #! ${stdenv.shell} -e
  459. $shellHook
  460. exec ${stdenv.shell}
  461. EOF
  462. chmod +x $out/bin/shell
  463. '';
  464. # Provide the dependencies in a development shell through the NODE_PATH environment variable
  465. inherit nodeDependencies;
  466. shellHook = lib.optionalString (dependencies != []) ''
  467. export NODE_PATH=${nodeDependencies}/lib/node_modules
  468. export PATH="${nodeDependencies}/bin:$PATH"
  469. '';
  470. };
  471. in
  472. {
  473. buildNodeSourceDist = lib.makeOverridable buildNodeSourceDist;
  474. buildNodePackage = lib.makeOverridable buildNodePackage;
  475. buildNodeDependencies = lib.makeOverridable buildNodeDependencies;
  476. buildNodeShell = lib.makeOverridable buildNodeShell;
  477. }