001package org.hl7.fhir.utilities.npm;
002
003import java.io.ByteArrayInputStream;
004import java.io.File;
005import java.io.FileInputStream;
006import java.io.FileNotFoundException;
007import java.io.IOException;
008import java.io.InputStream;
009import java.io.RandomAccessFile;
010import java.nio.channels.FileChannel;
011import java.nio.channels.FileLock;
012import java.sql.Timestamp;
013import java.text.ParseException;
014import java.text.SimpleDateFormat;
015import java.time.Instant;
016import java.util.ArrayList;
017import java.util.Arrays;
018import java.util.Collections;
019import java.util.Comparator;
020import java.util.Date;
021import java.util.HashMap;
022import java.util.List;
023import java.util.Locale;
024import java.util.Map;
025import java.util.Map.Entry;
026
027import org.apache.commons.io.FileUtils;
028import org.hl7.fhir.exceptions.FHIRException;
029import org.hl7.fhir.utilities.IniFile;
030import org.hl7.fhir.utilities.SimpleHTTPClient;
031import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
032import org.hl7.fhir.utilities.TextFile;
033import org.hl7.fhir.utilities.Utilities;
034import org.hl7.fhir.utilities.VersionUtilities;
035import org.hl7.fhir.utilities.json.model.JsonArray;
036import org.hl7.fhir.utilities.json.model.JsonElement;
037import org.hl7.fhir.utilities.json.model.JsonObject;
038import org.hl7.fhir.utilities.json.parser.JsonParser;
039import org.hl7.fhir.utilities.npm.NpmPackage.NpmPackageFolder;
040import org.hl7.fhir.utilities.npm.PackageList.PackageListEntry;
041import org.slf4j.Logger;
042import org.slf4j.LoggerFactory;
043
044/*
045  Copyright (c) 2011+, HL7, Inc.
046  All rights reserved.
047  
048  Redistribution and use in source and binary forms, with or without modification, 
049  are permitted provided that the following conditions are met:
050    
051   * Redistributions of source code must retain the above copyright notice, this 
052     list of conditions and the following disclaimer.
053   * Redistributions in binary form must reproduce the above copyright notice, 
054     this list of conditions and the following disclaimer in the documentation 
055     and/or other materials provided with the distribution.
056   * Neither the name of HL7 nor the names of its contributors may be used to 
057     endorse or promote products derived from this software without specific 
058     prior written permission.
059  
060  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
061  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
062  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
063  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
064  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
065  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
066  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
067  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
068  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
069  POSSIBILITY OF SUCH DAMAGE.
070  
071 */
072
073/**
074 * This is a package cache manager implementation that uses a local disk cache
075 *
076 * <p>
077 * API:
078 * <p>
079 * constructor
080 * getPackageUrl
081 * getPackageId
082 * findPackageCache
083 * addPackageToCache
084 *
085 * @author Grahame Grieve
086 */
087public class FilesystemPackageCacheManager extends BasePackageCacheManager implements IPackageCacheManager {
088
089
090
091  //  private static final String SECONDARY_SERVER = "http://local.fhir.org:8080/packages";
092  public static final String PACKAGE_REGEX = "^[a-zA-Z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+$";
093  public static final String PACKAGE_VERSION_REGEX = "^[A-Za-z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+\\#[A-Za-z0-9\\-\\_\\$]+(\\.[A-Za-z0-9\\-\\_\\$]+)*$";
094  public static final String PACKAGE_VERSION_REGEX_OPT = "^[A-Za-z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+(\\#[A-Za-z0-9\\-\\_]+(\\.[A-Za-z0-9\\-\\_]+)*)?$";
095  private static final Logger ourLog = LoggerFactory.getLogger(FilesystemPackageCacheManager.class);
096  private static final String CACHE_VERSION = "3"; // second version - see wiki page
097  private String cacheFolder;
098  private boolean progress = true;
099  private List<NpmPackage> temporaryPackages = new ArrayList<>();
100  private boolean buildLoaded = false;
101  private Map<String, String> ciList = new HashMap<String, String>();
102  private JsonArray buildInfo;
103  private boolean suppressErrors;
104 
105  /**
106   * Constructor
107   */
108  @Deprecated
109  public FilesystemPackageCacheManager(boolean userMode, int toolsVersion) throws IOException {
110    addPackageServer(PackageClient.PRIMARY_SERVER);
111    addPackageServer(PackageClient.SECONDARY_SERVER);
112
113    if (userMode)
114      cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "packages");
115    else
116      cacheFolder = Utilities.path("var", "lib", ".fhir", "packages");
117    if (!(new File(cacheFolder).exists()))
118      Utilities.createDirectory(cacheFolder);
119    if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists()))
120      TextFile.stringToFile("[cache]\r\nversion=" + CACHE_VERSION + "\r\n\r\n[urls]\r\n\r\n[local]\r\n\r\n", Utilities.path(cacheFolder, "packages.ini"), false);
121    createIniFile();
122  }
123
124  public FilesystemPackageCacheManager(boolean userMode) throws IOException {
125    addPackageServer(PackageClient.PRIMARY_SERVER);
126    addPackageServer(PackageClient.SECONDARY_SERVER);
127
128    if (userMode)
129      cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "packages");
130    else
131      cacheFolder = Utilities.path("var", "lib", ".fhir", "packages");
132    if (!(new File(cacheFolder).exists()))
133      Utilities.createDirectory(cacheFolder);
134    if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists()))
135      TextFile.stringToFile("[cache]\r\nversion=" + CACHE_VERSION + "\r\n\r\n[urls]\r\n\r\n[local]\r\n\r\n", Utilities.path(cacheFolder, "packages.ini"), false);
136    createIniFile();
137  }
138
139  public void loadFromFolder(String packagesFolder) throws IOException {
140    File[] files = new File(packagesFolder).listFiles();
141    if (files != null) {
142      for (File f : files) {
143        if (f.getName().endsWith(".tgz")) {
144          temporaryPackages.add(NpmPackage.fromPackage(new FileInputStream(f)));
145        }
146      }
147    }
148  }
149
150  public String getFolder() {
151    return cacheFolder;
152  }
153
154  private List<String> sorted(String[] keys) {
155    List<String> names = new ArrayList<String>();
156    for (String s : keys)
157      names.add(s);
158    Collections.sort(names);
159    return names;
160  }
161
162  private List<String> reverseSorted(String[] keys) {
163    Arrays.sort(keys, Collections.reverseOrder());
164    return Arrays.asList(keys);
165  }
166
167  private NpmPackage loadPackageInfo(String path) throws IOException {
168    NpmPackage pi = NpmPackage.fromFolder(path);
169    return pi;
170  }
171
172  private void clearCache() throws IOException {
173    for (File f : new File(cacheFolder).listFiles()) {
174      if (f.isDirectory()) {
175        new CacheLock(f.getName()).doWithLock(() -> {
176          Utilities.clearDirectory(f.getAbsolutePath());
177          try {
178            FileUtils.deleteDirectory(f);
179          } catch (Exception e1) {
180            try {
181              FileUtils.deleteDirectory(f);
182            } catch (Exception e2) {
183              // just give up
184            }
185          }
186          return null; // must return something
187        });
188      } else if (!f.getName().equals("packages.ini"))
189        FileUtils.forceDelete(f);
190    }
191    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
192    ini.removeSection("packages");
193    ini.save();
194  }
195
196  private void createIniFile() throws IOException {
197    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
198    boolean save = false;
199    String v = ini.getStringProperty("cache", "version");
200    if (!CACHE_VERSION.equals(v)) {
201      clearCache();
202      ini.setStringProperty("cache", "version", CACHE_VERSION, null);
203      ini.save();
204    }
205  }
206
207  private void checkValidVersionString(String version, String id) {
208    if (Utilities.noString(version)) {
209      throw new FHIRException("Cannot add package " + id + " to the package cache - a version must be provided");
210    }
211    if (version.startsWith("file:")) {
212      throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal in this context");
213    }
214    for (char ch : version.toCharArray()) {
215      if (!Character.isAlphabetic(ch) && !Character.isDigit(ch) && !Utilities.existsInList(ch, '.', '-', '$')) {
216        throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal (ch '" + ch + "'");
217      }
218    }
219  }
220
221  private void listSpecs(Map<String, String> specList, String server) throws IOException {
222    PackageClient pc = new PackageClient(server);
223    List<PackageInfo> matches = pc.search(null, null, null, false);
224    for (PackageInfo m : matches) {
225      if (!specList.containsKey(m.getId())) {
226        specList.put(m.getId(), m.getUrl());
227      }
228    }
229  }
230
231  protected InputStreamWithSrc loadFromPackageServer(String id, String version) {
232    InputStreamWithSrc retVal = super.loadFromPackageServer(id, version);
233    if (retVal != null) {
234      return retVal;
235    }
236
237    retVal = super.loadFromPackageServer(id, VersionUtilities.getMajMin(version)+".x");
238    if (retVal != null) {
239      return retVal;
240    }
241
242    // ok, well, we'll try the old way
243    return fetchTheOldWay(id, version);
244  }
245
246  public String getLatestVersion(String id) throws IOException {
247    for (String nextPackageServer : getPackageServers()) {
248      // special case:
249      if (!(Utilities.existsInList(id,CommonPackages.ID_PUBPACK, "hl7.terminology.r5") && PackageClient.PRIMARY_SERVER.equals(nextPackageServer))) {
250        PackageClient pc = new PackageClient(nextPackageServer);
251        try {
252          return pc.getLatestVersion(id);
253        } catch (IOException e) {
254          ourLog.info("Failed to determine latest version of package {} from server: {}", id, nextPackageServer);
255        }
256      }
257    }
258
259    return fetchVersionTheOldWay(id);
260  }
261
262  private NpmPackage loadPackageFromFile(String id, String folder) throws IOException {
263    File f = new File(Utilities.path(folder, id));
264    if (!f.exists()) {
265      throw new FHIRException("Package '" + id + "  not found in folder " + folder);
266    }
267    if (!f.isDirectory()) {
268      throw new FHIRException("File for '" + id + "  found in folder " + folder + ", not a folder");
269    }
270    File fp = new File(Utilities.path(folder, id, "package", "package.json"));
271    if (!fp.exists()) {
272      throw new FHIRException("Package '" + id + "  found in folder " + folder + ", but does not contain a package.json file in /package");
273    }
274    return NpmPackage.fromFolder(f.getAbsolutePath());
275  }
276
277  /**
278   * Clear the cache
279   *
280   * @throws IOException
281   */
282  public void clear() throws IOException {
283    clearCache();
284  }
285
286  // ========================= Utilities ============================================================================
287
288  /**
289   * Remove a particular package from the cache
290   *
291   * @param id
292   * @param ver
293   * @throws IOException
294   */
295  public void removePackage(String id, String ver) throws IOException {
296    new CacheLock(id + "#" + ver).doWithLock(() -> {
297      String f = Utilities.path(cacheFolder, id + "#" + ver);
298      File ff = new File(f);
299      if (ff.exists()) {
300        Utilities.clearDirectory(f);
301        IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
302        ini.removeProperty("packages", id + "#" + ver);
303        ini.save();
304        ff.delete();
305      }
306      return null;
307    });
308  }
309
310  /**
311   * Load the identified package from the cache - if it exists
312   * <p>
313   * This is for special purpose only (testing, control over speed of loading).
314   * Generally, use the loadPackage method
315   *
316   * @param id
317   * @param version
318   * @return
319   * @throws IOException
320   */
321  @Override
322  public NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException {
323    if (!Utilities.noString(version) && version.startsWith("file:")) {
324      return loadPackageFromFile(id, version.substring(5));
325    }
326
327    for (NpmPackage p : temporaryPackages) {
328      if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) {
329        return p;
330      }
331      if (p.name().equals(id) && Utilities.noString(version)) {
332        return p;
333      }
334    }
335    String foundPackage = null;
336    String foundVersion = null;
337    for (String f : reverseSorted(new File(cacheFolder).list())) {
338      File cf = new File(Utilities.path(cacheFolder, f));
339      if (cf.isDirectory()) {
340        if (f.equals(id + "#" + version) || (Utilities.noString(version) && f.startsWith(id + "#"))) {
341          return loadPackageInfo(Utilities.path(cacheFolder, f));
342        }
343        if (version != null && !version.equals("current") && (version.endsWith(".x") || Utilities.charCount(version, '.') < 2) && f.contains("#")) {
344          String[] parts = f.split("#");
345          if (parts[0].equals(id) && VersionUtilities.isMajMinOrLaterPatch((foundVersion!=null ? foundVersion : version),parts[1])) {
346            foundVersion = parts[1];
347            foundPackage = f;
348          }
349        }
350      }
351    }
352    if (foundPackage!=null) {
353      return loadPackageInfo(Utilities.path(cacheFolder, foundPackage));
354    }
355    if ("dev".equals(version))
356      return loadPackageFromCacheOnly(id, "current");
357    else
358      return null;
359  }
360
361  /**
362   * Add an already fetched package to the cache
363   */
364  @Override
365  public NpmPackage addPackageToCache(String id, String version, InputStream packageTgzInputStream, String sourceDesc) throws IOException {
366    checkValidVersionString(version, id);
367    if (progress) {
368      log("Installing " + id + "#" + (version == null ? "?" : version) + " to the package cache");
369      log("  Fetching:");
370    }
371
372    NpmPackage npm = NpmPackage.fromPackage(packageTgzInputStream, sourceDesc, true);
373
374    if (progress) {
375      log("");
376      logn("  Installing: ");
377    }
378    
379    if (!suppressErrors && npm.name() == null || id == null || !id.equalsIgnoreCase(npm.name())) {
380      if (!id.equals("hl7.fhir.r5.core") && !id.equals("hl7.fhir.us.immds")) {// temporary work around
381        throw new IOException("Attempt to import a mis-identified package. Expected " + id + ", got " + npm.name());
382      }
383    }
384    if (version == null)
385      version = npm.version();
386
387    String v = version;
388    return new CacheLock(id + "#" + version).doWithLock(() -> {
389      NpmPackage pck = null;
390      String packRoot = Utilities.path(cacheFolder, id + "#" + v);
391      try {
392        // ok, now we have a lock on it... check if something created it while we were waiting
393        if (!new File(packRoot).exists() || Utilities.existsInList(v, "current", "dev")) {
394          Utilities.createDirectory(packRoot);
395          try {
396            Utilities.clearDirectory(packRoot);
397          } catch (Throwable t) {
398            log("Unable to clear directory: "+packRoot+": "+t.getMessage()+" - this may cause problems later");
399          }
400
401          int i = 0;
402          int c = 0;
403          int size = 0;
404          for (Entry<String, NpmPackageFolder> e : npm.getFolders().entrySet()) {
405            String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey());
406            if (!(new File(dir).exists()))
407              Utilities.createDirectory(dir);
408            for (Entry<String, byte[]> fe : e.getValue().getContent().entrySet()) {
409              String fn = Utilities.path(dir, Utilities.cleanFileName(fe.getKey()));
410              byte[] cnt = fe.getValue();
411              TextFile.bytesToFile(cnt, fn);
412              size = size + cnt.length;
413              i++;
414              if (progress && i % 50 == 0) {
415                c++;
416                logn(".");
417                if (c == 120) {
418                  log("");
419                  logn("  ");
420                  c = 2;
421                }
422              }
423            }
424          }
425
426
427          IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
428          ini.setTimeStampFormat("yyyyMMddhhmmss");
429          ini.setTimestampProperty("packages", id + "#" + v, Timestamp.from(Instant.now()), null);
430          ini.setIntegerProperty("package-sizes", id + "#" + v, size, null);
431          ini.save();
432          if (progress)
433            log(" done.");
434        }
435        if (!id.equals(npm.getNpm().asString("name")) || !v.equals(npm.getNpm().asString("version"))) {
436          if (!id.equals(npm.getNpm().asString("name"))) {
437            npm.getNpm().add("original-name", npm.getNpm().asString("name"));
438            npm.getNpm().remove("name");
439            npm.getNpm().add("name", id);
440          }
441          if (!v.equals(npm.getNpm().asString("version"))) {
442            npm.getNpm().add("original-version", npm.getNpm().asString("version"));
443            npm.getNpm().remove("version");
444            npm.getNpm().add("version", v);
445          }
446          TextFile.stringToFile(JsonParser.compose(npm.getNpm(), true), Utilities.path(cacheFolder, id + "#" + v, "package", "package.json"), false);
447        }
448        pck = loadPackageInfo(packRoot);
449      } catch (Exception e) {
450        try {
451          // don't leave a half extracted package behind
452          log("Clean up package " + packRoot + " because installation failed: " + e.getMessage());
453          e.printStackTrace();
454          Utilities.clearDirectory(packRoot);
455          new File(packRoot).delete();
456        } catch (Exception ei) {
457          // nothing
458        }
459        throw e;
460      }
461      return pck;
462    });
463  }
464
465  private void log(String s) {
466    if (!silent) {
467      System.out.println(s);
468    }
469  }
470
471  private void logn(String s) {
472    if (!silent) {
473      System.out.print(s);
474    }
475  }
476
477  @Override
478  public String getPackageUrl(String packageId) throws IOException {
479    String result = super.getPackageUrl(packageId);
480    if (result == null) {
481      result = getPackageUrlFromBuildList(packageId);
482    }
483
484    return result;
485  }
486
487  public void listAllIds(Map<String, String> specList) throws IOException {
488    for (NpmPackage p : temporaryPackages) {
489      specList.put(p.name(), p.canonical());
490    }
491    for (String next : getPackageServers()) {
492      listSpecs(specList, next);
493    }
494    addCIBuildSpecs(specList);
495  }
496
497  @Override
498  public NpmPackage loadPackage(String id, String version) throws FHIRException, IOException {
499    //ok, try to resolve locally
500    if (!Utilities.noString(version) && version.startsWith("file:")) {
501      return loadPackageFromFile(id, version.substring(5));
502    }
503
504    if (version == null && id.contains("#")) {
505      version = id.substring(id.indexOf("#")+1);
506      id = id.substring(0, id.indexOf("#"));
507    }
508
509    if (version == null) {
510      try {
511        version = getLatestVersion(id);
512      } catch (Exception e) {
513        version = null;
514      }
515    }
516    NpmPackage p = loadPackageFromCacheOnly(id, version);
517    if (p != null) {
518      if ("current".equals(version)) {
519        p = checkCurrency(id, p);
520      }
521      if (p != null)
522        return p;
523    }
524
525    if ("dev".equals(version)) {
526      p = loadPackageFromCacheOnly(id, "current");
527      p = checkCurrency(id, p);
528      if (p != null)
529        return p;
530      version = "current";
531    }
532
533    // nup, don't have it locally (or it's expired)
534    FilesystemPackageCacheManager.InputStreamWithSrc source;
535    if ("current".equals(version) || (version!= null && version.startsWith("current$"))) {
536      // special case - fetch from ci-build server
537      source = loadFromCIBuild(id, version.startsWith("current$") ? version.substring(8) : null);
538    } else {
539      source = loadFromPackageServer(id, version);
540    }
541    if (source == null) {
542      throw new FHIRException("Unable to find package "+id+"#"+version);
543    }
544    return addPackageToCache(id, source.version, source.stream, source.url);
545  }
546
547  private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException {
548    try {
549      SimpleHTTPClient http = new SimpleHTTPClient();
550      HTTPResult res = http.get(source);
551      res.checkThrowException();
552      return new ByteArrayInputStream(res.getContent());
553    } catch (Exception e) {
554      if (optional)
555        return null;
556      else
557        throw new FHIRException("Unable to fetch: "+e.getMessage(), e);
558    }
559  }
560
561  private InputStreamWithSrc loadFromCIBuild(String id, String branch) throws IOException {
562    checkBuildLoaded();
563    if (ciList.containsKey(id)) {
564      if (branch == null) {
565        InputStream stream;
566        try {
567          stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "package.tgz"), false);
568        } catch (Exception e) {
569           stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "branches", "main", "package.tgz"), false);          
570        }
571        return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "package.tgz"), "current");
572      } else {
573        InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), false);
574        return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), "current$"+branch);
575      }
576    } else if (id.startsWith("hl7.fhir.r5")) {
577      InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("http://build.fhir.org", id + ".tgz"), false);
578      return new InputStreamWithSrc(stream, Utilities.pathURL("http://build.fhir.org", id + ".tgz"), "current");
579    } else {
580      throw new FHIRException("The package '" + id + "' has no entry on the current build server ("+ciList.toString()+")");
581    }
582  }
583
584  private String getPackageUrlFromBuildList(String packageId) throws IOException {
585    checkBuildLoaded();
586    for (JsonObject o : buildInfo.asJsonObjects()) {
587      if (packageId.equals(o.asString("package-id"))) {
588        return o.asString("url");
589      }
590    }
591    return null;
592  }
593
594  private void addCIBuildSpecs(Map<String, String> specList) throws IOException {
595    checkBuildLoaded();
596    for (JsonElement n : buildInfo) {
597      JsonObject o = (JsonObject) n;
598      if (!specList.containsKey(o.asString("package-id"))) {
599        specList.put(o.asString("package-id"), o.asString("url"));
600      }
601    }
602  }
603
604  @Override
605  public String getPackageId(String canonicalUrl) throws IOException {
606    String retVal = findCanonicalInLocalCache(canonicalUrl);
607    
608    if(retVal == null) {
609      retVal = super.getPackageId(canonicalUrl);
610    }
611    
612    if (retVal == null) {
613      retVal = getPackageIdFromBuildList(canonicalUrl);
614    }
615
616    return retVal;
617  }
618
619
620  public String findCanonicalInLocalCache(String canonicalUrl) {
621    try {
622      for (String pf : listPackages()) {
623        if (new File(Utilities.path(cacheFolder, pf, "package", "package.json")).exists()) {
624          JsonObject npm = JsonParser.parseObjectFromFile(Utilities.path(cacheFolder, pf, "package", "package.json"));
625          if (canonicalUrl.equals(npm.asString("canonical"))) {
626            return npm.asString("name");
627          }
628        }
629      }
630    } catch (IOException e) {
631    }
632    return null;
633  }
634
635  // ========================= Package Mgmt API =======================================================================
636
637  private String getPackageIdFromBuildList(String canonical) throws IOException {
638    if (canonical == null) {
639      return null;
640    }
641    checkBuildLoaded();
642    if (buildInfo != null) {
643      for (JsonElement n : buildInfo) {
644        JsonObject o = (JsonObject) n;
645        if (canonical.equals(o.asString("url"))) {
646          return o.asString("package-id");
647        }
648      }
649      for (JsonElement n : buildInfo) {
650        JsonObject o = (JsonObject) n;
651        if (o.asString("url").startsWith(canonical + "/ImplementationGuide/")) {
652          return o.asString("package-id");
653        }
654      }
655    }
656    return null;
657  }
658
659  private NpmPackage checkCurrency(String id, NpmPackage p) throws IOException {
660    checkBuildLoaded();
661    // special case: current versions roll over, and we have to check their currency
662    try {
663      String url = ciList.get(id);
664      JsonObject json = JsonParser.parseObjectFromUrl(Utilities.pathURL(url, "package.manifest.json"));
665      String currDate = json.asString("date");
666      String packDate = p.date();
667      if (!currDate.equals(packDate)) {
668        return null; // nup, we need a new copy
669      }
670    } catch (Exception e) {
671    }
672    return p;
673  }
674
675  private boolean checkBuildLoaded() {
676    if (buildLoaded)
677      return true;
678    try {
679      loadFromBuildServer();
680    } catch (Exception e) {
681      log("Error connecting to build server - running without build (" + e.getMessage() + ")");
682      e.printStackTrace();
683    }
684    return false;
685  }
686
687  private void loadFromBuildServer() throws IOException {
688    SimpleHTTPClient http = new SimpleHTTPClient();
689    http.trustAllhosts();
690    HTTPResult res = http.get("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis());
691    res.checkThrowException();
692
693    buildInfo = (JsonArray) JsonParser.parse(TextFile.bytesToString(res.getContent()));
694
695    List<BuildRecord> builds = new ArrayList<>();
696
697    for (JsonElement n : buildInfo) {
698      JsonObject o = (JsonObject) n;
699      if (o.has("url") && o.has("package-id") && o.asString("package-id").contains(".")) {
700        String u = o.asString("url");
701        if (u.contains("/ImplementationGuide/"))
702          u = u.substring(0, u.indexOf("/ImplementationGuide/"));
703        builds.add(new BuildRecord(u, o.asString("package-id"), getRepo(o.asString("repo")), readDate(o.asString("date"))));
704      }
705    }
706    Collections.sort(builds, new BuildRecordSorter());
707    for (BuildRecord bld : builds) {
708      if (!ciList.containsKey(bld.getPackageId())) {
709        ciList.put(bld.getPackageId(), "https://build.fhir.org/ig/" + bld.getRepo());
710      }
711    }
712    buildLoaded = true; // whether it succeeds or not
713  }
714
715  private String getRepo(String path) {
716    String[] p = path.split("\\/");
717    return p[0] + "/" + p[1];
718  }
719
720  private Date readDate(String s) {
721    SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM, yyyy HH:mm:ss Z", new Locale("en", "US"));
722    try {
723      return sdf.parse(s);
724    } catch (ParseException e) {
725      e.printStackTrace();
726      return new Date();
727    }
728  }
729
730  // ----- the old way, from before package server, while everything gets onto the package server
731  private InputStreamWithSrc fetchTheOldWay(String id, String v) {
732    String url = getUrlForPackage(id);
733    if (url == null) {
734      try {
735        url = getPackageUrlFromBuildList(id);
736      } catch (Exception e) {
737        url = null;
738      }
739    }
740    if (url == null) {
741      throw new FHIRException("Unable to resolve package id " + id + "#" + v);
742    }
743    if (url.contains("/ImplementationGuide/")) {
744      url = url.substring(0, url.indexOf("/ImplementationGuide/"));
745    }
746    String pu = Utilities.pathURL(url, "package-list.json");
747    String aurl = pu;
748    PackageList pl;
749    try {
750      pl = PackageList.fromUrl(pu);
751    } catch (Exception e) {
752      String pv = Utilities.pathURL(url, v, "package.tgz");
753      try {
754        aurl = pv;
755        InputStreamWithSrc src = new InputStreamWithSrc(fetchFromUrlSpecific(pv, false), pv, v);
756        return src;
757      } catch (Exception e1) {
758        throw new FHIRException("Error fetching package directly (" + pv + "), or fetching package list for " + id + " from " + pu + ": " + e1.getMessage(), e1);
759      }
760    }
761    if (!id.equals(pl.pid()))
762      throw new FHIRException("Package ids do not match in " + pu + ": " + id + " vs " + pl.pid());
763    for (PackageListEntry vo : pl.versions()) {
764      if (v.equals(vo.version())) {
765        aurl = Utilities.pathURL(vo.path(), "package.tgz");
766        String u = Utilities.pathURL(vo.path(), "package.tgz");
767        return new InputStreamWithSrc(fetchFromUrlSpecific(u, true), u, v);
768      }
769    }
770
771    return null;
772  }
773
774
775  // ---------- Current Build SubSystem --------------------------------------------------------------------------------------
776
777  private String fetchVersionTheOldWay(String id) throws IOException {
778    String url = getUrlForPackage(id);
779    if (url == null) {
780      try {
781        url = getPackageUrlFromBuildList(id);
782      } catch (Exception e) {
783        url = null;
784      }
785    }
786    if (url == null) {
787      throw new FHIRException("Unable to resolve package id " + id);
788    }
789    PackageList pl = PackageList.fromUrl(Utilities.pathURL(url, "package-list.json"));
790    if (!id.equals(pl.pid()))
791      throw new FHIRException("Package ids do not match in " + pl.source() + ": " + id + " vs " + pl.pid());
792    for (PackageListEntry vo : pl.versions()) {
793      if (vo.current()) {
794        return vo.version();
795      }
796    }
797
798    return null;
799  }
800
801  private String getUrlForPackage(String id) {
802    if (CommonPackages.ID_XVER.equals(id)) {
803      return "http://fhir.org/packages/hl7.fhir.xver-extensions";
804    }
805    return null;
806  }
807
808  public List<String> listPackages() {
809    List<String> res = new ArrayList<>();
810    for (File f : new File(cacheFolder).listFiles()) {
811      if (f.isDirectory() && f.getName().contains("#")) {
812        res.add(f.getName());
813      }
814    }
815    return res;
816  }
817
818  /**
819   * if you don't provide and implementation of this interface, the PackageCacheManager will use the web directly.
820   * <p>
821   * You can use this interface to
822   *
823   * @author graha
824   */
825  public interface INetworkServices {
826
827    InputStream resolvePackage(String packageId, String version);
828  }
829
830  public interface CacheLockFunction<T> {
831    T get() throws IOException;
832  }
833
834  public class BuildRecordSorter implements Comparator<BuildRecord> {
835
836    @Override
837    public int compare(BuildRecord arg0, BuildRecord arg1) {
838      return arg1.date.compareTo(arg0.date);
839    }
840  }
841
842  public class BuildRecord {
843
844    private String url;
845    private String packageId;
846    private String repo;
847    private Date date;
848
849    public BuildRecord(String url, String packageId, String repo, Date date) {
850      super();
851      this.url = url;
852      this.packageId = packageId;
853      this.repo = repo;
854      this.date = date;
855    }
856
857    public String getUrl() {
858      return url;
859    }
860
861    public String getPackageId() {
862      return packageId;
863    }
864
865    public String getRepo() {
866      return repo;
867    }
868
869    public Date getDate() {
870      return date;
871    }
872
873
874  }
875
876
877
878  public class VersionHistory {
879    private String id;
880    private String canonical;
881    private String current;
882    private Map<String, String> versions = new HashMap<>();
883
884    public String getCanonical() {
885      return canonical;
886    }
887
888    public String getCurrent() {
889      return current;
890    }
891
892    public Map<String, String> getVersions() {
893      return versions;
894    }
895
896    public String getId() {
897      return id;
898    }
899  }
900
901  public class PackageEntry {
902
903    private byte[] bytes;
904    private String name;
905
906    public PackageEntry(String name) {
907      this.name = name;
908    }
909
910    public PackageEntry(String name, byte[] bytes) {
911      this.name = name;
912      this.bytes = bytes;
913    }
914  }
915
916  public class CacheLock {
917
918    private final File lockFile;
919
920    public CacheLock(String name) throws IOException {
921      this.lockFile = new File(cacheFolder, name + ".lock");
922      if (!lockFile.isFile()) {
923        TextFile.stringToFile("", lockFile);
924      }
925    }
926
927    public <T> T doWithLock(CacheLockFunction<T> f) throws FileNotFoundException, IOException {
928      try (FileChannel channel = new RandomAccessFile(lockFile, "rw").getChannel()) {
929        final FileLock fileLock = channel.lock();
930        T result = null;
931        try {
932          result = f.get();
933        } finally {
934          fileLock.release();
935        }
936        if (!lockFile.delete()) {
937          lockFile.deleteOnExit();
938        }
939        return result;
940      }
941    }
942  }
943
944  public boolean packageExists(String id, String ver) throws IOException {
945    if (packageInstalled(id, ver)) {
946      return true;
947    }
948    for (String s : getPackageServers()) {
949      if (new PackageClient(s).exists(id, ver)) {
950        return true;
951      }
952    }
953    return false;
954  }
955
956  public boolean packageInstalled(String id, String version) {
957    for (NpmPackage p : temporaryPackages) {
958      if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) {
959        return true;
960      }
961      if (p.name().equals(id) && Utilities.noString(version)) {
962        return true;
963      }
964    }
965
966    for (String f : sorted(new File(cacheFolder).list())) {
967      if (f.equals(id + "#" + version) || (Utilities.noString(version) && f.startsWith(id + "#"))) {
968        return true;
969      }
970    }
971    if ("dev".equals(version))
972      return packageInstalled(id, "current");
973    else
974      return false;
975  }
976
977  public boolean isSuppressErrors() {
978    return suppressErrors;
979  }
980
981  public void setSuppressErrors(boolean suppressErrors) {
982    this.suppressErrors = suppressErrors;
983  }
984
985  
986}