Source code

001package org.hl7.fhir.validation;
002
003import java.io.ByteArrayInputStream;
004import java.io.ByteArrayOutputStream;
005import java.io.File;
006import java.io.FileInputStream;
007import java.io.IOException;
008import java.io.InputStream;
009import java.util.ArrayList;
010import java.util.Arrays;
011import java.util.HashMap;
012import java.util.List;
013import java.util.Map;
014import java.util.zip.ZipEntry;
015import java.util.zip.ZipInputStream;
016
017import org.hl7.fhir.convertors.factory.VersionConvertorFactory_10_50;
018import org.hl7.fhir.convertors.factory.VersionConvertorFactory_14_50;
019import org.hl7.fhir.convertors.factory.VersionConvertorFactory_30_50;
020import org.hl7.fhir.convertors.factory.VersionConvertorFactory_40_50;
021import org.hl7.fhir.convertors.factory.VersionConvertorFactory_43_50;
022import org.hl7.fhir.exceptions.FHIRException;
023import org.hl7.fhir.r5.context.SimpleWorkerContext;
024import org.hl7.fhir.r5.elementmodel.Manager;
025import org.hl7.fhir.r5.formats.JsonParser;
026import org.hl7.fhir.r5.formats.XmlParser;
027import org.hl7.fhir.r5.model.Constants;
028import org.hl7.fhir.r5.model.ImplementationGuide;
029import org.hl7.fhir.r5.model.Resource;
030import org.hl7.fhir.r5.utils.structuremap.StructureMapUtilities;
031import org.hl7.fhir.utilities.IniFile;
032import org.hl7.fhir.utilities.SimpleHTTPClient;
033import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
034import org.hl7.fhir.utilities.TextFile;
035import org.hl7.fhir.utilities.Utilities;
036import org.hl7.fhir.utilities.VersionUtilities;
037import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
038import org.hl7.fhir.utilities.npm.NpmPackage;
039import org.hl7.fhir.utilities.turtle.Turtle;
040import org.hl7.fhir.validation.cli.utils.Common;
041import org.hl7.fhir.validation.cli.utils.VersionSourceInformation;
042
043import lombok.Getter;
044
045public class IgLoader {
046
047  private static final String[] IGNORED_EXTENSIONS = {"md", "css", "js", "png", "gif", "jpg", "html", "tgz", "pack", "zip"};
048  private static final String[] EXEMPT_FILES = {"spec.internals", "version.info", "schematron.zip", "package.json"};
049  private static final int SCAN_HEADER_SIZE = 2048;
050
051  @Getter private final FilesystemPackageCacheManager packageCacheManager;
052  @Getter private final SimpleWorkerContext context;
053  @Getter private final String version;
054  @Getter private final boolean isDebug;
055
056  public IgLoader(FilesystemPackageCacheManager packageCacheManager,
057                  SimpleWorkerContext context,
058                  String theVersion) {
059      this(packageCacheManager, context, theVersion, false);
060  }
061
062  public IgLoader(FilesystemPackageCacheManager packageCacheManager,
063                  SimpleWorkerContext context,
064                  String theVersion,
065                  boolean isDebug) {
066      this.packageCacheManager = packageCacheManager;
067      this.context = context;
068      this.version = theVersion;
069      this.isDebug = isDebug;
070  }
071
072  /**
073   *
074   * @param igs
075   * @param binaries
076   * @param src Source of the IG
077   *
078   * @param recursive
079   * @throws IOException
080   * @throws FHIRException
081   *
082   * @see IgLoader#loadIgSource(String, boolean, boolean) loadIgSource for detailed description of the src parameter
083   */
084  public void loadIg(List<ImplementationGuide> igs,
085                     Map<String, byte[]> binaries,
086                     String src,
087                     boolean recursive) throws IOException, FHIRException {
088
089    final String explicitFhirVersion;
090    final String srcPackage;
091    if (src.startsWith("[") && src.indexOf(']', 1) > 1) {
092      explicitFhirVersion = src.substring(1,src.indexOf(']', 1));
093      srcPackage = src.substring(src.indexOf(']',1) + 1);
094      if (!VersionUtilities.isSupportedVersion(explicitFhirVersion)) {
095        throw new FHIRException("Unsupported FHIR Version: " + explicitFhirVersion + " valid versions are " + VersionUtilities.listSupportedVersions());
096      }
097    } else {
098      explicitFhirVersion = null;
099      srcPackage = src;
100    }
101
102    NpmPackage npm = srcPackage.matches(FilesystemPackageCacheManager.PACKAGE_VERSION_REGEX_OPT) && !new File(srcPackage).exists() ? getPackageCacheManager().loadPackage(srcPackage, null) : null;
103    if (npm != null) {
104      for (String s : npm.dependencies()) {
105        if (!getContext().getLoadedPackages().contains(s)) {
106          if (!VersionUtilities.isCorePackage(s)) {
107            loadIg(igs, binaries, s, false);
108          }
109        }
110      }
111      System.out.print("  Load " + srcPackage);
112      if (!srcPackage.contains("#")) {
113        System.out.print("#" + npm.version());
114      }
115      int count = getContext().loadFromPackage(npm, ValidatorUtils.loaderForVersion(npm.fhirVersion()));
116      System.out.println(" - " + count + " resources (" + getContext().clock().milestone() + ")");
117    } else {
118      System.out.print("  Load " + srcPackage);
119      String canonical = null;
120      int count = 0;
121      Map<String, byte[]> source = loadIgSource(srcPackage, recursive, true);
122      String version = Constants.VERSION;
123      if (getVersion() != null) {
124        version = getVersion();
125      }
126      if (source.containsKey("version.info")) {
127        version = readInfoVersion(source.get("version.info"));
128      }
129      if (explicitFhirVersion != null) {
130        version = explicitFhirVersion;
131      }
132
133      for (Map.Entry<String, byte[]> t : source.entrySet()) {
134        String fn = t.getKey();
135        if (!exemptFile(fn)) {
136          Resource r = loadFileWithErrorChecking(version, t, fn);
137          if (r != null) {
138            count++;
139            getContext().cacheResource(r);
140            if (r instanceof ImplementationGuide) {
141              canonical = ((ImplementationGuide) r).getUrl();
142              igs.add((ImplementationGuide) r);
143              if (canonical.contains("/ImplementationGuide/")) {
144                Resource r2 = r.copy();
145                ((ImplementationGuide) r2).setUrl(canonical.substring(0, canonical.indexOf("/ImplementationGuide/")));
146                getContext().cacheResource(r2);
147              }
148            }
149          }
150        }
151      }
152      if (canonical != null) {
153        ValidatorUtils.grabNatives(binaries, source, canonical);
154      }
155      System.out.println(" - " + count + " resources (" + getContext().clock().milestone() + ")");
156    }
157  }
158
159  /**
160   *
161   * @param source
162   * @param opName
163   * @param asIg
164   * @return
165   * @throws FHIRException
166   * @throws IOException
167   *
168   *    * @see IgLoader#loadIgSource(String, boolean, boolean) loadIgSource for detailed description of the src parameter
169   */
170
171  public Content loadContent(String source, String opName, boolean asIg) throws FHIRException, IOException {
172    Map<String, byte[]> s = loadIgSource(source, false, asIg);
173    Content res = new Content();
174    if (s.size() != 1)
175      throw new FHIRException("Unable to find resource " + source + " to " + opName);
176    for (Map.Entry<String, byte[]> t : s.entrySet()) {
177      res.focus = t.getValue();
178      if (t.getKey().endsWith(".json"))
179        res.cntType = Manager.FhirFormat.JSON;
180      else if (t.getKey().endsWith(".xml"))
181        res.cntType = Manager.FhirFormat.XML;
182      else if (t.getKey().endsWith(".ttl"))
183        res.cntType = Manager.FhirFormat.TURTLE;
184      else if (t.getKey().endsWith(".shc"))
185        res.cntType = Manager.FhirFormat.SHC;
186      else if (t.getKey().endsWith(".txt") || t.getKey().endsWith(".map"))
187        res.cntType = Manager.FhirFormat.TEXT;
188      else
189        throw new FHIRException("Todo: Determining resource type is not yet done");
190    }
191    return res;
192  }
193
194  /**
195   *
196   * @param src can be one of the following:
197   *      <br> - a canonical url for an ig - this will be converted to a package id and loaded into the cache
198   *      <br> - a package id for an ig - this will be loaded into the cache
199   *      <br> - a direct reference to a package ("package.tgz") - this will be extracted by the cache manager, but not put in the cache
200   *      <br> - a folder containing resources - these will be loaded directly
201   * @param recursive if true and src resolves to a folder, recursively find and load IgSources from that directory
202   * @param explore should be true if we're trying to load an -ig parameter, and false if we're loading source
203   *
204   * @return
205   * @throws FHIRException
206   * @throws IOException
207   */
208  public Map<String, byte[]> loadIgSource(String src,
209                                          boolean recursive,
210                                          boolean explore) throws FHIRException, IOException {
211    //
212    if (Common.isNetworkPath(src)) {
213      String v = null;
214      if (src.contains("|")) {
215        v = src.substring(src.indexOf("|") + 1);
216        src = src.substring(0, src.indexOf("|"));
217      }
218      String pid = explore ? getPackageCacheManager().getPackageId(src) : null;
219      if (!Utilities.noString(pid))
220        return fetchByPackage(pid + (v == null ? "" : "#" + v), false);
221      else
222        return fetchFromUrl(src + (v == null ? "" : "|" + v), explore);
223    }
224
225    File f = new File(Utilities.path(src));
226    if (f.exists()) {
227      if (f.isDirectory() && new File(Utilities.path(src, "package.tgz")).exists()) {
228        FileInputStream stream = new FileInputStream(Utilities.path(src, "package.tgz"));
229        try {
230          return loadPackage(stream, Utilities.path(src, "package.tgz"), false);
231        } finally {
232          stream.close();
233        }
234      }
235      if (f.isDirectory() && new File(Utilities.path(src, "igpack.zip")).exists()) {
236        FileInputStream stream = new FileInputStream(Utilities.path(src, "igpack.zip"));
237        try {
238          return readZip(stream);
239        } finally {
240          stream.close();
241        }
242      }
243      if (f.isDirectory() && new File(Utilities.path(src, "validator.pack")).exists()) {
244        FileInputStream stream = new FileInputStream(Utilities.path(src, "validator.pack"));
245        try {
246          return readZip(stream);
247        } finally {
248          stream.close();
249        }
250      }
251      if (f.isDirectory()) {
252        return scanDirectory(f, recursive);
253      }
254      FileInputStream stream = new FileInputStream(src);
255      try {
256        if (src.endsWith(".tgz")) {
257          Map<String, byte[]> res = loadPackage(stream, src, false);
258          return res;
259        }
260        if (src.endsWith(".pack")) {
261          return readZip(stream);
262        }
263        if (src.endsWith("igpack.zip")) {
264          return readZip(stream);
265        }
266      } finally {
267        stream.close();
268      }
269
270      Manager.FhirFormat fmt = ResourceChecker.checkIsResource(getContext(), isDebug(), TextFile.fileToBytes(f), src, true);
271      if (fmt != null) {
272        Map<String, byte[]> res = new HashMap<String, byte[]>();
273        res.put(Utilities.changeFileExt(src, "." + fmt.getExtension()), TextFile.fileToBytesNCS(src));
274        return res;
275      }
276    } else if ((src.matches(FilesystemPackageCacheManager.PACKAGE_REGEX) || src.matches(FilesystemPackageCacheManager.PACKAGE_VERSION_REGEX)) && !src.endsWith(".zip") && !src.endsWith(".tgz")) {
277      return fetchByPackage(src, false);
278    }
279    throw new FHIRException("Unable to find/resolve/read " + (explore ? "-ig " : "") + src);
280  }
281
282  public void scanForIgVersion(String src,
283                               boolean recursive,
284                               VersionSourceInformation versions) throws Exception {
285    Map<String, byte[]> source = loadIgSourceForVersion(src, recursive, true, versions);
286    if (source != null) {
287      if (source.containsKey("version.info")) {
288        versions.see(readInfoVersion(source.get("version.info")), "version.info in " + src);
289      } else if (source.size() == 1) {
290        for (byte[] v : source.values()) {
291          scanForFhirVersion(versions, src, v);
292        }
293      }
294    }
295  }
296
297  public void scanForVersions(List<String> sources, VersionSourceInformation versions) throws FHIRException, IOException {
298    List<String> refs = new ArrayList<String>();
299    ValidatorUtils.parseSources(sources, refs, context);
300    for (String ref : refs) {
301      Content cnt = loadContent(ref, "validate", false);      
302      scanForFhirVersion(versions, ref, cnt.focus);
303    }
304  }
305
306  private void scanForFhirVersion(VersionSourceInformation versions, String ref, byte[] cnt) throws IOException {
307    String s = TextFile.bytesToString(cnt.length > SCAN_HEADER_SIZE ? Arrays.copyOfRange(cnt, 0, SCAN_HEADER_SIZE) : cnt).trim();
308    try {
309      int i = s.indexOf("fhirVersion");
310      if (i > 1) {
311        boolean xml = s.charAt(i) == '<';
312        i = find(s, i, '"');
313        if (!xml) {
314          i = find(s, i+1, '"');          
315        }
316        if (i > 0) {
317          int j = find(s, i+1, '"');
318          if (j > 0) {
319            String v = s.substring(i+1, j);
320            if (VersionUtilities.isSemVer(v)) {
321              versions.see(VersionUtilities.getMajMin(v), "fhirVersion in " + ref);
322              return;
323            }
324          }
325        }
326        i = find(s, i, '\'');
327        if (!xml) {
328          i = find(s, i+1, '\'');          
329        }
330        if (i > 0) {
331          int j = find(s, i+1, '\'');
332          if (j > 0) {
333            String v = s.substring(i, j);
334            if (VersionUtilities.isSemVer(v)) {
335              versions.see(VersionUtilities.getMajMin(v), "fhirVersion in " + ref);
336              return;
337            }
338          }
339        }
340      }
341    } catch (Exception e) {
342      // nothing
343    }
344    if (s.contains("http://hl7.org/fhir/3.0")) {
345      versions.see("3.0", "Profile in " + ref);
346      return;
347    }
348    if (s.contains("http://hl7.org/fhir/1.0")) {
349      versions.see("1.0", "Profile in " + ref);
350      return;
351    }
352    if (s.contains("http://hl7.org/fhir/4.0")) {
353      versions.see("4.0", "Profile in " + ref);
354      return;
355    }
356    if (s.contains("http://hl7.org/fhir/1.4")) {
357      versions.see("1.4", "Profile in " + ref);
358      return;
359    }
360  }
361
362  private int find(String s, int i, char c) {
363    while (i < s.length() && s.charAt(i) != c) {
364      i++;
365    }
366    return i == s.length() ? -1 : i;
367  }
368
369  protected Map<String, byte[]> readZip(InputStream stream) throws IOException {
370    Map<String, byte[]> res = new HashMap<>();
371    ZipInputStream zip = new ZipInputStream(stream);
372    ZipEntry ze;
373    while ((ze = zip.getNextEntry()) != null) {
374      String name = ze.getName();
375      ByteArrayOutputStream b = new ByteArrayOutputStream();
376      int n;
377      byte[] buf = new byte[1024];
378      while ((n = ((InputStream) zip).read(buf, 0, 1024)) > -1) {
379        b.write(buf, 0, n);
380      }
381      res.put(name, b.toByteArray());
382      zip.closeEntry();
383    }
384    zip.close();
385    return res;
386  }
387
388  private String loadPackageForVersion(InputStream stream) throws FHIRException, IOException {
389    return NpmPackage.fromPackage(stream).fhirVersion();
390  }
391
392  private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException, IOException {
393    try {
394      SimpleHTTPClient http = new SimpleHTTPClient();
395      HTTPResult res = http.get(source + "?nocache=" + System.currentTimeMillis());
396      res.checkThrowException();
397      return new ByteArrayInputStream(res.getContent());
398    } catch (IOException e) {
399      if (optional)
400        return null;
401      else
402        throw e;
403    }
404  }
405
406  private Map<String, byte[]> loadIgSourceForVersion(String src,
407                                                     boolean recursive,
408                                                     boolean explore,
409                                                     VersionSourceInformation versions) throws FHIRException, IOException {
410    if (Common.isNetworkPath(src)) {
411      String v = null;
412      if (src.contains("|")) {
413        v = src.substring(src.indexOf("|") + 1);
414        src = src.substring(0, src.indexOf("|"));
415      }
416      String pid = getPackageCacheManager().getPackageId(src);
417      if (!Utilities.noString(pid)) {
418        versions.see(fetchVersionByPackage(pid + (v == null ? "" : "#" + v)), "Package " + src);
419        return null;
420      } else {
421        return fetchVersionFromUrl(src + (v == null ? "" : "|" + v), explore, versions);
422      }
423    }
424
425    File f = new File(Utilities.path(src));
426    if (f.exists()) {
427      if (f.isDirectory() && new File(Utilities.path(src, "package.tgz")).exists()) {
428        versions.see(loadPackageForVersion(new FileInputStream(Utilities.path(src, "package.tgz"))), "Package " + src);
429        return null;
430      }
431      if (f.isDirectory() && new File(Utilities.path(src, "igpack.zip")).exists())
432        return readZip(new FileInputStream(Utilities.path(src, "igpack.zip")));
433      if (f.isDirectory() && new File(Utilities.path(src, "validator.pack")).exists())
434        return readZip(new FileInputStream(Utilities.path(src, "validator.pack")));
435      if (f.isDirectory())
436        return scanDirectory(f, recursive);
437      if (src.endsWith(".tgz")) {
438        versions.see(loadPackageForVersion(new FileInputStream(src)), "Package " + src);
439        return null;
440      }
441      if (src.endsWith(".pack"))
442        return readZip(new FileInputStream(src));
443      if (src.endsWith("igpack.zip"))
444        return readZip(new FileInputStream(src));
445      Manager.FhirFormat fmt = ResourceChecker.checkIsResource(getContext(), isDebug(), TextFile.fileToBytes(f), src, true);
446      if (fmt != null) {
447        Map<String, byte[]> res = new HashMap<String, byte[]>();
448        res.put(Utilities.changeFileExt(src, "." + fmt.getExtension()), TextFile.fileToBytesNCS(src));
449        return res;
450      }
451    } else if ((src.matches(FilesystemPackageCacheManager.PACKAGE_REGEX) || src.matches(FilesystemPackageCacheManager.PACKAGE_VERSION_REGEX)) && !src.endsWith(".zip") && !src.endsWith(".tgz")) {
452      versions.see(fetchVersionByPackage(src), "Package " + src);
453      return null;
454    }
455    throw new FHIRException("Unable to find/resolve/read -ig " + src);
456  }
457
458
459  private Map<String, byte[]> fetchByPackage(String src, boolean loadInContext) throws FHIRException, IOException {
460    String id = src;
461    String version = null;
462    if (src.contains("#")) {
463      id = src.substring(0, src.indexOf("#"));
464      version = src.substring(src.indexOf("#") + 1);
465    }
466    if (version == null) {
467      version = getPackageCacheManager().getLatestVersion(id);
468    }
469    NpmPackage pi;
470    if (version == null) {
471      pi = getPackageCacheManager().loadPackageFromCacheOnly(id);
472      if (pi != null)
473        System.out.println("   ... Using version " + pi.version());
474    } else
475      pi = getPackageCacheManager().loadPackageFromCacheOnly(id, version);
476    if (pi == null) {
477      return resolvePackage(id, version, loadInContext);
478    } else
479      return loadPackage(pi, loadInContext);
480  }
481
482  private Map<String, byte[]> loadPackage(InputStream stream, String name, boolean loadInContext) throws FHIRException, IOException {
483    return loadPackage(NpmPackage.fromPackage(stream), loadInContext);
484  }
485
486  public Map<String, byte[]> loadPackage(NpmPackage pi, boolean loadInContext) throws FHIRException, IOException {
487    Map<String, byte[]> res = new HashMap<String, byte[]>();
488    for (String s : pi.dependencies()) {
489      if (s.endsWith(".x") && s.length() > 2) {
490        String packageMajorMinor = s.substring(0, s.length() - 2);
491        boolean found = false;
492        for (int i = 0; i < getContext().getLoadedPackages().size() && !found; ++i) {
493          String loadedPackage = getContext().getLoadedPackages().get(i);
494          if (loadedPackage.startsWith(packageMajorMinor)) {
495            found = true;
496          }
497        }
498        if (found)
499          continue;
500      }
501      if (!getContext().getLoadedPackages().contains(s)) {
502        if (!VersionUtilities.isCorePackage(s)) {
503          System.out.println("+  .. load IG from " + s);
504          res.putAll(fetchByPackage(s, loadInContext));
505        }
506      }
507    }
508
509    if (loadInContext) {
510//      getContext().getLoadedPackages().add(pi.name() + "#" + pi.version());
511      getContext().loadFromPackage(pi, ValidatorUtils.loaderForVersion(pi.fhirVersion()));
512    }
513    for (String s : pi.listResources("CodeSystem", "ConceptMap", "ImplementationGuide", "CapabilityStatement", "SearchParameter", "Conformance", "StructureMap", "ValueSet", "StructureDefinition")) {
514      res.put(s, TextFile.streamToBytes(pi.load("package", s)));
515    }
516    String ini = "[FHIR]\r\nversion=" + pi.fhirVersion() + "\r\n";
517    res.put("version.info", ini.getBytes());
518    return res;
519  }
520
521  private Map<String, byte[]> resolvePackage(String id, String v, boolean loadInContext) throws FHIRException, IOException {
522    NpmPackage pi = getPackageCacheManager().loadPackage(id, v);
523    if (pi != null && v == null)
524      System.out.println("   ... Using version " + pi.version());
525    return loadPackage(pi,  loadInContext);
526  }
527
528  private String readInfoVersion(byte[] bs) throws IOException {
529    String is = TextFile.bytesToString(bs);
530    is = is.trim();
531    IniFile ini = new IniFile(new ByteArrayInputStream(TextFile.stringToBytes(is, false)));
532    return ini.getStringProperty("FHIR", "version");
533  }
534
535  private byte[] fetchFromUrlSpecific(String source, String contentType, boolean optional, List<String> errors) throws FHIRException, IOException {
536    try {
537      SimpleHTTPClient http = new SimpleHTTPClient();
538      try {
539        // try with cache-busting option and then try withhout in case the server doesn't support that
540        HTTPResult res = http.get(source + "?nocache=" + System.currentTimeMillis(), contentType);
541        res.checkThrowException();
542        return res.getContent();
543      } catch (Exception e) {
544        HTTPResult res = http.get(source, contentType);
545        res.checkThrowException();
546        return res.getContent();
547      }
548    } catch (IOException e) {
549      if (errors != null) {
550        errors.add("Error accessing " + source + ": " + e.getMessage());
551      }
552      if (optional)
553        return null;
554      else
555        throw e;
556    }
557  }
558
559  private Map<String, byte[]> fetchVersionFromUrl(String src,
560                                                  boolean explore,
561                                                  VersionSourceInformation versions) throws FHIRException, IOException {
562    if (src.endsWith(".tgz")) {
563      versions.see(loadPackageForVersion(fetchFromUrlSpecific(src, false)), "From Package " + src);
564      return null;
565    }
566    if (src.endsWith(".pack"))
567      return readZip(fetchFromUrlSpecific(src, false));
568    if (src.endsWith("igpack.zip"))
569      return readZip(fetchFromUrlSpecific(src, false));
570
571    InputStream stream = null;
572    if (explore) {
573      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "package.tgz"), true);
574      if (stream != null) {
575        versions.see(loadPackageForVersion(stream), "From Package at " + src);
576        return null;
577      }
578      // todo: these options are deprecated - remove once all IGs have been rebuilt post R4 technical correction
579      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "igpack.zip"), true);
580      if (stream != null)
581        return readZip(stream);
582      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "validator.pack"), true);
583      if (stream != null)
584        return readZip(stream);
585      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "validator.pack"), true);
586      //// -----
587    }
588
589    // ok, having tried all that... now we'll just try to access it directly
590    byte[] cnt;
591    if (stream == null)
592      cnt = fetchFromUrlSpecific(src, "application/json", true, null);
593    else
594      cnt = TextFile.streamToBytes(stream);
595
596    Manager.FhirFormat fmt = ResourceChecker.checkIsResource(getContext(), isDebug(), cnt, src, true);
597    if (fmt != null) {
598      Map<String, byte[]> res = new HashMap<String, byte[]>();
599      res.put(Utilities.changeFileExt(src, "." + fmt.getExtension()), cnt);
600      return res;
601    }
602    String fn = Utilities.path("[tmp]", "fetch-resource-error-content.bin");
603    TextFile.bytesToFile(cnt, fn);
604    System.out.println("Error Fetching " + src);
605    System.out.println("Some content was found, saved to " + fn);
606    System.out.println("1st 100 bytes = " + presentForDebugging(cnt));
607    throw new FHIRException("Unable to find/resolve/read " + (explore ? "-ig " : "") + src);
608  }
609
610  private String fetchVersionByPackage(String src) throws FHIRException, IOException {
611    String id = src;
612    String version = null;
613    if (src.contains("#")) {
614      id = src.substring(0, src.indexOf("#"));
615      version = src.substring(src.indexOf("#") + 1);
616    }
617    if (version == null) {
618      version = getPackageCacheManager().getLatestVersion(id);
619    }
620    NpmPackage pi = null;
621    if (version == null) {
622      pi = getPackageCacheManager().loadPackageFromCacheOnly(id);
623      if (pi != null)
624        System.out.println("   ... Using version " + pi.version());
625    } else
626      pi = getPackageCacheManager().loadPackageFromCacheOnly(id, version);
627    if (pi == null) {
628      return resolvePackageForVersion(id, version);
629    } else {
630      return pi.fhirVersion();
631    }
632  }
633
634  private Map<String, byte[]> fetchFromUrl(String src, boolean explore) throws FHIRException, IOException {
635    if (src.endsWith(".tgz"))
636      return loadPackage(fetchFromUrlSpecific(src, false), src, false);
637    if (src.endsWith(".pack"))
638      return readZip(fetchFromUrlSpecific(src, false));
639    if (src.endsWith("igpack.zip"))
640      return readZip(fetchFromUrlSpecific(src, false));
641
642    InputStream stream = null;
643    if (explore) {
644      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "package.tgz"), true);
645      if (stream != null)
646        return loadPackage(stream, Utilities.pathURL(src, "package.tgz"), false);
647      // todo: these options are deprecated - remove once all IGs have been rebuilt post R4 technical correction
648      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "igpack.zip"), true);
649      if (stream != null)
650        return readZip(stream);
651      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "validator.pack"), true);
652      if (stream != null)
653        return readZip(stream);
654      stream = fetchFromUrlSpecific(Utilities.pathURL(src, "validator.pack"), true);
655      //// -----
656    }
657
658    // ok, having tried all that... now we'll just try to access it directly
659    byte[] cnt;
660    List<String> errors = new ArrayList<>();
661    if (stream != null) {
662      cnt = TextFile.streamToBytes(stream);
663    } else {
664      cnt = fetchFromUrlSpecific(src, "application/json", true, errors);
665      if (cnt == null) {
666        cnt = fetchFromUrlSpecific(src, "application/xml", true, errors);
667      }
668    }
669    if (cnt == null) {
670      throw new FHIRException("Unable to fetch content from " + src + " (" + errors.toString() + ")");
671
672    }
673    Manager.FhirFormat fmt = checkFormat(cnt, src);
674    if (fmt != null) {
675      Map<String, byte[]> res = new HashMap<>();
676      res.put(Utilities.changeFileExt(src, "." + fmt.getExtension()), cnt);
677      return res;
678    }
679    throw new FHIRException("Unable to read content from " + src + ": cannot determine format");
680  }
681
682  private boolean isIgnoreFile(File ff) {
683    if (ff.getName().startsWith(".") || ff.getAbsolutePath().contains(".git")) {
684      return true;
685    }
686    return Utilities.existsInList(Utilities.getFileExtension(ff.getName()).toLowerCase(), IGNORED_EXTENSIONS);
687  }
688
689  private Map<String, byte[]> scanDirectory(File f, boolean recursive) throws IOException {
690    Map<String, byte[]> res = new HashMap<>();
691    for (File ff : f.listFiles()) {
692      if (ff.isDirectory() && recursive) {
693        res.putAll(scanDirectory(ff, true));
694      } else if (!ff.isDirectory() && !isIgnoreFile(ff)) {
695        Manager.FhirFormat fmt = ResourceChecker.checkIsResource(getContext(), isDebug(), TextFile.fileToBytes(ff), ff.getAbsolutePath(), true);
696        if (fmt != null) {
697          res.put(Utilities.changeFileExt(ff.getName(), "." + fmt.getExtension()), TextFile.fileToBytes(ff.getAbsolutePath()));
698        }
699      }
700    }
701    return res;
702  }
703
704  private String resolvePackageForVersion(String id, String v) throws FHIRException, IOException {
705    NpmPackage pi = getPackageCacheManager().loadPackage(id, v);
706    return pi.fhirVersion();
707  }
708
709  private String presentForDebugging(byte[] cnt) {
710    StringBuilder b = new StringBuilder();
711    for (int i = 0; i < Integer.min(cnt.length, 50); i++) {
712      b.append(Integer.toHexString(cnt[i]));
713    }
714    return b.toString();
715  }
716
717  private Manager.FhirFormat checkFormat(byte[] cnt, String filename) {
718    System.out.println("   ..Detect format for " + filename);
719    try {
720      org.hl7.fhir.utilities.json.parser.JsonParser.parseObject(cnt);
721      return Manager.FhirFormat.JSON;
722    } catch (Exception e) {
723      log("Not JSON: " + e.getMessage());
724    }
725    try {
726      ValidatorUtils.parseXml(cnt);
727      return Manager.FhirFormat.XML;
728    } catch (Exception e) {
729      log("Not XML: " + e.getMessage());
730    }
731    try {
732      new Turtle().parse(TextFile.bytesToString(cnt));
733      return Manager.FhirFormat.TURTLE;
734    } catch (Exception e) {
735      log("Not Turtle: " + e.getMessage());
736    }
737    try {
738      new StructureMapUtilities(getContext(), null, null).parse(TextFile.bytesToString(cnt), null);
739      return Manager.FhirFormat.TEXT;
740    } catch (Exception e) {
741      log("Not Text: " + e.getMessage());
742    }
743    log("     .. not a resource: " + filename);
744    return null;
745  }
746
747  private boolean exemptFile(String fn) {
748    return Utilities.existsInList(fn, EXEMPT_FILES);
749  }
750
751  protected Resource loadFileWithErrorChecking(String version, Map.Entry<String, byte[]> t, String fn) {
752    log("* load file: " + fn);
753    Resource r = null;
754    try {
755      r = loadResourceByVersion(version, t.getValue(), fn);
756      log(" .. success");
757    } catch (Exception e) {
758      if (!isDebug()) {
759        System.out.print("* load file: " + fn);
760      }
761      System.out.println(" - ignored due to error: " + (e.getMessage() == null ? " (null - NPE)" : e.getMessage()));
762      if (isDebug() || ((e.getMessage() != null && e.getMessage().contains("cannot be cast")))) {
763        e.printStackTrace();
764      }
765    }
766    return r;
767  }
768
769  public Resource loadResourceByVersion(String fhirVersion, byte[] content, String fn) throws IOException, FHIRException {
770    Resource r;
771    if (fhirVersion.startsWith("3.0")) {
772      org.hl7.fhir.dstu3.model.Resource res;
773      if (fn.endsWith(".xml") && !fn.endsWith("template.xml"))
774        res = new org.hl7.fhir.dstu3.formats.XmlParser().parse(new ByteArrayInputStream(content));
775      else if (fn.endsWith(".json") && !fn.endsWith("template.json"))
776        res = new org.hl7.fhir.dstu3.formats.JsonParser().parse(new ByteArrayInputStream(content));
777      else if (fn.endsWith(".txt") || fn.endsWith(".map"))
778        res = new org.hl7.fhir.dstu3.utils.StructureMapUtilities(null).parse(new String(content));
779      else
780        throw new FHIRException("Unsupported format for " + fn);
781      r = VersionConvertorFactory_30_50.convertResource(res);
782    } else if (fhirVersion.startsWith("4.0")) {
783      org.hl7.fhir.r4.model.Resource res;
784      if (fn.endsWith(".xml") && !fn.endsWith("template.xml"))
785        res = new org.hl7.fhir.r4.formats.XmlParser().parse(new ByteArrayInputStream(content));
786      else if (fn.endsWith(".json") && !fn.endsWith("template.json"))
787        res = new org.hl7.fhir.r4.formats.JsonParser().parse(new ByteArrayInputStream(content));
788      else if (fn.endsWith(".txt") || fn.endsWith(".map"))
789        res = new org.hl7.fhir.r4.utils.StructureMapUtilities(null).parse(new String(content), fn);
790      else
791        throw new FHIRException("Unsupported format for " + fn);
792      r = VersionConvertorFactory_40_50.convertResource(res);
793    } else if (fhirVersion.startsWith("4.3")) {
794      org.hl7.fhir.r4b.model.Resource res;
795      if (fn.endsWith(".xml") && !fn.endsWith("template.xml"))
796        res = new org.hl7.fhir.r4b.formats.XmlParser().parse(new ByteArrayInputStream(content));
797      else if (fn.endsWith(".json") && !fn.endsWith("template.json"))
798        res = new org.hl7.fhir.r4b.formats.JsonParser().parse(new ByteArrayInputStream(content));
799      else if (fn.endsWith(".txt") || fn.endsWith(".map"))
800        res = new org.hl7.fhir.r4b.utils.structuremap.StructureMapUtilities(null).parse(new String(content), fn);
801      else
802        throw new FHIRException("Unsupported format for " + fn);
803      r = VersionConvertorFactory_43_50.convertResource(res);
804    } else if (fhirVersion.startsWith("1.4")) {
805      org.hl7.fhir.dstu2016may.model.Resource res;
806      if (fn.endsWith(".xml") && !fn.endsWith("template.xml"))
807        res = new org.hl7.fhir.dstu2016may.formats.XmlParser().parse(new ByteArrayInputStream(content));
808      else if (fn.endsWith(".json") && !fn.endsWith("template.json"))
809        res = new org.hl7.fhir.dstu2016may.formats.JsonParser().parse(new ByteArrayInputStream(content));
810      else
811        throw new FHIRException("Unsupported format for " + fn);
812      r = VersionConvertorFactory_14_50.convertResource(res);
813    } else if (fhirVersion.startsWith("1.0")) {
814      org.hl7.fhir.dstu2.model.Resource res;
815      if (fn.endsWith(".xml") && !fn.endsWith("template.xml"))
816        res = new org.hl7.fhir.dstu2.formats.JsonParser().parse(new ByteArrayInputStream(content));
817      else if (fn.endsWith(".json") && !fn.endsWith("template.json"))
818        res = new org.hl7.fhir.dstu2.formats.JsonParser().parse(new ByteArrayInputStream(content));
819      else
820        throw new FHIRException("Unsupported format for " + fn);
821      r = VersionConvertorFactory_10_50.convertResource(res, new org.hl7.fhir.convertors.misc.IGR2ConvertorAdvisor5());
822    } else if (fhirVersion.startsWith("5.0")) {
823      if (fn.endsWith(".xml") && !fn.endsWith("template.xml"))
824        r = new XmlParser().parse(new ByteArrayInputStream(content));
825      else if (fn.endsWith(".json") && !fn.endsWith("template.json"))
826        r = new JsonParser().parse(new ByteArrayInputStream(content));
827      else if (fn.endsWith(".txt"))
828        r = new StructureMapUtilities(getContext(), null, null).parse(TextFile.bytesToString(content), fn);
829      else if (fn.endsWith(".map"))
830        r = new StructureMapUtilities(null).parse(new String(content), fn);
831      else
832        throw new FHIRException("Unsupported format for " + fn);
833    } else
834      throw new FHIRException("Unsupported version " + fhirVersion);
835    return r;
836  }
837
838  private void log(String s) {
839    if (isDebug()) System.out.println(s);
840  }
841}