Source code

001package org.hl7.fhir.r5.terminologies;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import static org.apache.commons.lang3.StringUtils.isNotBlank;
035
036import java.io.FileNotFoundException;
037import java.io.IOException;
038
039/*
040 * Copyright (c) 2011+, HL7, Inc
041 * All rights reserved.
042 * 
043 * Redistribution and use in source and binary forms, with or without modification,
044 * are permitted provided that the following conditions are met:
045 * 
046 * Redistributions of source code must retain the above copyright notice, this
047 * list of conditions and the following disclaimer.
048 * Redistributions in binary form must reproduce the above copyright notice,
049 * this list of conditions and the following disclaimer in the documentation
050 * and/or other materials provided with the distribution.
051 * Neither the name of HL7 nor the names of its contributors may be used to
052 * endorse or promote products derived from this software without specific
053 * prior written permission.
054 * 
055 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
056 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
057 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
058 * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
059 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
060 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
061 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
062 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
063 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
064 * POSSIBILITY OF SUCH DAMAGE.
065 * 
066 */
067
068import java.util.ArrayList;
069import java.util.Collection;
070import java.util.HashMap;
071import java.util.HashSet;
072import java.util.List;
073import java.util.Map;
074import java.util.Set;
075
076import org.apache.commons.lang3.NotImplementedException;
077import org.hl7.fhir.exceptions.FHIRException;
078import org.hl7.fhir.exceptions.FHIRFormatError;
079import org.hl7.fhir.exceptions.NoTerminologyServiceException;
080import org.hl7.fhir.exceptions.TerminologyServiceException;
081import org.hl7.fhir.r5.context.IWorkerContext;
082import org.hl7.fhir.r5.model.BooleanType;
083import org.hl7.fhir.r5.model.CodeSystem;
084import org.hl7.fhir.r5.model.CodeSystem.CodeSystemContentMode;
085import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionComponent;
086import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionDesignationComponent;
087import org.hl7.fhir.r5.model.CodeSystem.ConceptPropertyComponent;
088import org.hl7.fhir.r5.model.CodeSystem.PropertyComponent;
089import org.hl7.fhir.r5.model.CodeSystem.PropertyType;
090import org.hl7.fhir.r5.model.DataType;
091import org.hl7.fhir.r5.model.DateTimeType;
092import org.hl7.fhir.r5.model.Enumerations.FilterOperator;
093import org.hl7.fhir.r5.model.Extension;
094import org.hl7.fhir.r5.model.Factory;
095import org.hl7.fhir.r5.model.Parameters;
096import org.hl7.fhir.r5.model.Parameters.ParametersParameterComponent;
097import org.hl7.fhir.r5.model.PrimitiveType;
098import org.hl7.fhir.r5.model.UriType;
099import org.hl7.fhir.r5.model.ValueSet;
100import org.hl7.fhir.r5.model.ValueSet.ConceptReferenceComponent;
101import org.hl7.fhir.r5.model.ValueSet.ConceptReferenceDesignationComponent;
102import org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent;
103import org.hl7.fhir.r5.model.ValueSet.ConceptSetFilterComponent;
104import org.hl7.fhir.r5.model.ValueSet.ValueSetComposeComponent;
105import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionComponent;
106import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionContainsComponent;
107import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionParameterComponent;
108import org.hl7.fhir.r5.utils.ToolingExtensions;
109import org.hl7.fhir.utilities.Utilities;
110
111public class ValueSetExpanderSimple extends ValueSetWorker implements ValueSetExpander {
112
113  public class PropertyFilter implements IConceptFilter {
114
115    private ConceptSetFilterComponent filter;
116    private PropertyComponent property;
117
118    public PropertyFilter(ConceptSetFilterComponent fc, PropertyComponent propertyDefinition) {
119      this.filter = fc;
120      this.property = propertyDefinition;
121    }
122
123    @Override
124    public boolean includeConcept(CodeSystem cs, ConceptDefinitionComponent def) {
125      ConceptPropertyComponent pc = getPropertyForConcept(def);
126      if (pc != null) {
127        String v = pc.getValue().isPrimitive() ? pc.getValue().primitiveValue() : null;
128        switch (filter.getOp()) {
129        case DESCENDENTOF: throw fail("not supported yet: "+filter.getOp().toCode());
130        case EQUAL: return filter.getValue().equals(v);
131        case EXISTS: throw fail("not supported yet: "+filter.getOp().toCode());
132        case GENERALIZES: throw fail("not supported yet: "+filter.getOp().toCode());
133        case IN: throw fail("not supported yet: "+filter.getOp().toCode());
134        case ISA: throw fail("not supported yet: "+filter.getOp().toCode());
135        case ISNOTA: throw fail("not supported yet: "+filter.getOp().toCode());
136        case NOTIN: throw fail("not supported yet: "+filter.getOp().toCode());
137        case NULL: throw fail("not supported yet: "+filter.getOp().toCode());
138        case REGEX: throw fail("not supported yet: "+filter.getOp().toCode());
139        default:
140          throw fail("Shouldn't get here");        
141        }            
142      } else if (property.getType() == PropertyType.BOOLEAN && filter.getOp() == FilterOperator.EQUAL) {
143        return "false".equals(filter.getValue()); 
144      } else {
145        return false;
146      }
147    }
148
149    private ConceptPropertyComponent getPropertyForConcept(ConceptDefinitionComponent def) {
150      for (ConceptPropertyComponent pc : def.getProperty()) {
151        if (pc.getCode().equals(property.getCode())) {
152          return pc;
153        }
154      }
155      return null;
156    }
157
158  }
159
160  public class AllConceptsFilter implements IConceptFilter {
161
162    @Override
163    public boolean includeConcept(CodeSystem cs, ConceptDefinitionComponent def) {
164      return true;
165    }
166  }
167
168  public interface IConceptFilter {
169
170    boolean includeConcept(CodeSystem cs, ConceptDefinitionComponent def);
171
172  }
173
174  private List<ValueSetExpansionContainsComponent> codes = new ArrayList<ValueSet.ValueSetExpansionContainsComponent>();
175  private List<ValueSetExpansionContainsComponent> roots = new ArrayList<ValueSet.ValueSetExpansionContainsComponent>();
176  private Map<String, ValueSetExpansionContainsComponent> map = new HashMap<String, ValueSet.ValueSetExpansionContainsComponent>();
177  private IWorkerContext context;
178  private boolean canBeHeirarchy = true;
179  private boolean includeAbstract = true;
180  private Set<String> excludeKeys = new HashSet<String>();
181  private Set<String> excludeSystems = new HashSet<String>();
182  private ValueSet focus;
183  private int maxExpansionSize = 500;
184  private List<String> allErrors = new ArrayList<>();
185
186  private int total;
187  private boolean checkCodesWhenExpanding;
188
189  public ValueSetExpanderSimple(IWorkerContext context) {
190    super();
191    this.context = context;
192  }
193
194  public ValueSetExpanderSimple(IWorkerContext context, List<String> allErrors) {
195    super();
196    this.context = context;
197    this.allErrors = allErrors;
198  }
199
200  public void setMaxExpansionSize(int theMaxExpansionSize) {
201    maxExpansionSize = theMaxExpansionSize;
202  }
203  
204  private ValueSetExpansionContainsComponent addCode(String system, String code, String display, ValueSetExpansionContainsComponent parent, List<ConceptDefinitionDesignationComponent> designations, Parameters expParams, boolean isAbstract, boolean inactive, List<ValueSet> filters) {
205 
206    if (filters != null && !filters.isEmpty() && !filterContainsCode(filters, system, code))
207      return null;
208    ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
209    n.setSystem(system);
210    n.setCode(code);
211    if (isAbstract)
212      n.setAbstract(true);
213    if (inactive)
214      n.setInactive(true);
215
216    if (expParams.getParameterBool("includeDesignations") && designations != null) {
217      for (ConceptDefinitionDesignationComponent t : designations) {
218        ToolingExtensions.addLanguageTranslation(n, t.getLanguage(), t.getValue());
219      }
220    }
221    ConceptDefinitionDesignationComponent t = expParams.hasLanguage() ? getMatchingLang(designations, expParams.getLanguage()) : null;
222    if (t == null)
223      n.setDisplay(display);
224    else
225      n.setDisplay(t.getValue());
226
227    String s = key(n);
228    if (map.containsKey(s) || excludeKeys.contains(s)) {
229      canBeHeirarchy = false;
230    } else {
231      codes.add(n);
232      map.put(s, n);
233      total++;
234    }
235    if (canBeHeirarchy && parent != null) {
236      parent.getContains().add(n);
237    } else {
238      roots.add(n);
239    }
240    return n;
241  }
242
243  private boolean filterContainsCode(List<ValueSet> filters, String system, String code) {
244    for (ValueSet vse : filters)
245      if (expansionContainsCode(vse.getExpansion().getContains(), system, code))
246        return true;
247    return false;
248  }
249
250  private boolean expansionContainsCode(List<ValueSetExpansionContainsComponent> contains, String system, String code) {
251    for (ValueSetExpansionContainsComponent cc : contains) {
252      if (system.equals(cc.getSystem()) && code.equals(cc.getCode()))
253        return true;
254      if (expansionContainsCode(cc.getContains(), system, code))
255        return true;
256    }
257    return false;
258  }
259
260  private ConceptDefinitionDesignationComponent getMatchingLang(List<ConceptDefinitionDesignationComponent> list, String lang) {
261    for (ConceptDefinitionDesignationComponent t : list)
262      if (t.getLanguage().equals(lang))
263        return t;
264    for (ConceptDefinitionDesignationComponent t : list)
265      if (t.getLanguage().startsWith(lang))
266        return t;
267    return null;
268  }
269
270  private void addCodeAndDescendents(ValueSetExpansionContainsComponent focus, ValueSetExpansionContainsComponent parent, Parameters expParams, List<ValueSet> filters)  throws FHIRException {
271    focus.checkNoModifiers("Expansion.contains", "expanding");
272    ValueSetExpansionContainsComponent np = addCode(focus.getSystem(), focus.getCode(), focus.getDisplay(), parent, 
273         convert(focus.getDesignation()), expParams, focus.getAbstract(), focus.getInactive(), filters);
274    for (ValueSetExpansionContainsComponent c : focus.getContains())
275      addCodeAndDescendents(focus, np, expParams, filters);
276  }
277  
278  private List<ConceptDefinitionDesignationComponent> convert(List<ConceptReferenceDesignationComponent> designations) {
279    List<ConceptDefinitionDesignationComponent> list = new ArrayList<ConceptDefinitionDesignationComponent>();
280    for (ConceptReferenceDesignationComponent d : designations) {
281      ConceptDefinitionDesignationComponent n = new ConceptDefinitionDesignationComponent();
282      n.setLanguage(d.getLanguage());
283      n.setUse(d.getUse());
284      n.setValue(d.getValue());
285      list.add(n);
286    }
287    return list;
288  }
289
290  private void addCodeAndDescendents(CodeSystem cs, String system, ConceptDefinitionComponent def, ValueSetExpansionContainsComponent parent, Parameters expParams, List<ValueSet> filters, ConceptDefinitionComponent exclusion, IConceptFilter filterFunc)  throws FHIRException {
291    def.checkNoModifiers("Code in Code System", "expanding");
292    if (exclusion != null) {
293      if (exclusion.getCode().equals(def.getCode()))
294        return; // excluded.
295    }
296    if (!CodeSystemUtilities.isDeprecated(cs, def, false)) {
297      ValueSetExpansionContainsComponent np = null;
298      boolean abs = CodeSystemUtilities.isNotSelectable(cs, def);
299      boolean inc = CodeSystemUtilities.isInactive(cs, def);
300      if ((includeAbstract || !abs)  && filterFunc.includeConcept(cs, def)) {
301        np = addCode(system, def.getCode(), def.getDisplay(), parent, def.getDesignation(), expParams, abs, inc, filters);
302      }
303      for (ConceptDefinitionComponent c : def.getConcept()) {
304        addCodeAndDescendents(cs, system, c, np, expParams, filters, exclusion, filterFunc);
305      }
306      if (def.hasUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK)) {
307        List<ConceptDefinitionComponent> children = (List<ConceptDefinitionComponent>) def.getUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK);
308        for (ConceptDefinitionComponent c : children)
309          addCodeAndDescendents(cs, system, c, np, expParams, filters, exclusion, filterFunc);
310      }
311    } else {
312      for (ConceptDefinitionComponent c : def.getConcept()) {
313        addCodeAndDescendents(cs, system, c, null, expParams, filters, exclusion, filterFunc);
314      }
315      if (def.hasUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK)) {
316        List<ConceptDefinitionComponent> children = (List<ConceptDefinitionComponent>) def.getUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK);
317        for (ConceptDefinitionComponent c : children)
318          addCodeAndDescendents(cs, system, c, null, expParams, filters, exclusion, filterFunc);
319      }
320    }
321
322  }
323
324  private void addCodes(ValueSetExpansionComponent expand, List<ValueSetExpansionParameterComponent> params, Parameters expParams, List<ValueSet> filters) throws ETooCostly, FHIRException {
325    if (expand != null) {
326      if (expand.getContains().size() > maxExpansionSize)
327        throw failCostly("Too many codes to display (>" + Integer.toString(expand.getContains().size()) + ")");
328      for (ValueSetExpansionParameterComponent p : expand.getParameter()) {
329        if (!existsInParams(params, p.getName(), p.getValue()))
330          params.add(p);
331      }
332
333      copyImportContains(expand.getContains(), null, expParams, filters);
334    }
335  }
336
337  private void excludeCode(String theSystem, String theCode) {
338    ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
339    n.setSystem(theSystem);
340    n.setCode(theCode);
341    String s = key(n);
342    excludeKeys.add(s);
343  }
344
345  private void excludeCodes(ConceptSetComponent exc, List<ValueSetExpansionParameterComponent> params, String ctxt) throws FHIRException {
346    exc.checkNoModifiers("Compose.exclude", "expanding");
347    if (exc.hasSystem() && exc.getConcept().size() == 0 && exc.getFilter().size() == 0) {
348      excludeSystems.add(exc.getSystem());
349    }
350
351    if (exc.hasValueSet())
352      throw fail("Processing Value set references in exclude is not yet done in "+ctxt);
353    // importValueSet(imp.getValue(), params, expParams);
354
355    CodeSystem cs = context.fetchCodeSystem(exc.getSystem());
356    if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(exc.getSystem())) {
357      ValueSetExpansionOutcome vse = context.expandVS(exc, false);
358      ValueSet valueset = vse.getValueset();
359      if (valueset == null)
360        throw failTSE("Error Expanding ValueSet: "+vse.getError());
361      excludeCodes(valueset.getExpansion(), params);
362      return;
363    }
364
365    for (ConceptReferenceComponent c : exc.getConcept()) {
366      excludeCode(exc.getSystem(), c.getCode());
367    }
368
369    if (exc.getFilter().size() > 0)
370      throw fail("not done yet - multiple filters");
371  }
372
373
374
375  private void excludeCodes(ValueSetExpansionComponent expand, List<ValueSetExpansionParameterComponent> params) {
376    for (ValueSetExpansionContainsComponent c : expand.getContains()) {
377      excludeCode(c.getSystem(), c.getCode());
378    }
379  }
380
381  private boolean existsInParams(List<ValueSetExpansionParameterComponent> params, String name, DataType value) {
382    for (ValueSetExpansionParameterComponent p : params) {
383      if (p.getName().equals(name) && PrimitiveType.compareDeep(p.getValue(), value, false))
384        return true;
385    }
386    return false;
387  }
388
389  @Override
390  public ValueSetExpansionOutcome expand(ValueSet source, Parameters expParams) {
391    allErrors.clear();
392    try {
393      return expandInternal(source, expParams);
394    } catch (NoTerminologyServiceException e) {
395      // well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set
396      // that might fail too, but it might not, later.
397      return new ValueSetExpansionOutcome(e.getMessage(), TerminologyServiceErrorClass.NOSERVICE, allErrors);
398    } catch (Exception e) {
399      // well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set
400      // that might fail too, but it might not, later.
401      return new ValueSetExpansionOutcome(e.getMessage(), TerminologyServiceErrorClass.UNKNOWN, allErrors);
402    }
403  }
404  
405  public ValueSetExpansionOutcome expandInternal(ValueSet source, Parameters expParams) throws FHIRException, FileNotFoundException, ETooCostly, IOException {
406      return doExpand(source, expParams);
407  }
408
409  public ValueSetExpansionOutcome doExpand(ValueSet source, Parameters expParams) throws FHIRException, ETooCostly, FileNotFoundException, IOException {
410    if (expParams == null)
411      expParams = makeDefaultExpansion();
412    source.checkNoModifiers("ValueSet", "expanding");
413    focus = source.copy();
414    focus.setExpansion(new ValueSet.ValueSetExpansionComponent());
415    focus.getExpansion().setTimestampElement(DateTimeType.now());
416    focus.getExpansion().setIdentifier(Factory.createUUID()); 
417    for (ParametersParameterComponent p : expParams.getParameter()) {
418      if (Utilities.existsInList(p.getName(), "includeDesignations", "excludeNested"))
419        focus.getExpansion().addParameter().setName(p.getName()).setValue(p.getValue());
420    }
421
422    if (source.hasCompose())
423      handleCompose(source.getCompose(), focus.getExpansion(), expParams, source.getUrl(), focus.getExpansion().getExtension());
424
425    if (canBeHeirarchy) {
426      for (ValueSetExpansionContainsComponent c : roots) {
427        focus.getExpansion().getContains().add(c);
428      }
429    } else {
430      for (ValueSetExpansionContainsComponent c : codes) {
431        if (map.containsKey(key(c)) && (includeAbstract || !c.getAbstract())) { // we may have added abstract codes earlier while we still thought it might be heirarchical, but later we gave up, so now ignore them
432          focus.getExpansion().getContains().add(c);
433          c.getContains().clear(); // make sure any heirarchy is wiped
434        }
435      }
436    }
437
438    if (total > 0) {
439      focus.getExpansion().setTotal(total);
440    }
441
442    return new ValueSetExpansionOutcome(focus);
443  }
444
445  private Parameters makeDefaultExpansion() {
446    Parameters res = new Parameters();
447    res.addParameter("excludeNested", true);
448    res.addParameter("includeDesignations", false);
449    return res;
450  }
451
452  private ConceptDefinitionComponent getConceptForCode(List<ConceptDefinitionComponent> clist, String code) {
453    for (ConceptDefinitionComponent c : clist) {
454      if (code.equals(c.getCode()))
455        return c;
456      ConceptDefinitionComponent v = getConceptForCode(c.getConcept(), code);
457      if (v != null)
458        return v;
459    }
460    return null;
461  }
462
463  private void handleCompose(ValueSetComposeComponent compose, ValueSetExpansionComponent exp, Parameters expParams, String ctxt, List<Extension> extensions)
464      throws ETooCostly, FileNotFoundException, IOException, FHIRException {
465    compose.checkNoModifiers("ValueSet.compose", "expanding");
466    // Exclude comes first because we build up a map of things to exclude
467    for (ConceptSetComponent inc : compose.getExclude())
468      excludeCodes(inc, exp.getParameter(), ctxt);
469    canBeHeirarchy = !expParams.getParameterBool("excludeNested") && excludeKeys.isEmpty() && excludeSystems.isEmpty();
470    includeAbstract = !expParams.getParameterBool("excludeNotForUI");
471    boolean first = true;
472    for (ConceptSetComponent inc : compose.getInclude()) {
473      if (first == true)
474        first = false;
475      else
476        canBeHeirarchy = false;
477      includeCodes(inc, exp, expParams, canBeHeirarchy, extensions);
478    }
479  }
480
481  private ValueSet importValueSet(String value, ValueSetExpansionComponent exp, Parameters expParams) throws ETooCostly, TerminologyServiceException, FileNotFoundException, IOException, FHIRFormatError {
482    if (value == null)
483      throw fail("unable to find value set with no identity");
484    ValueSet vs = context.fetchResource(ValueSet.class, value);
485    if (vs == null) {
486      if (context.fetchResource(CodeSystem.class, value) != null) {
487        throw fail("Cannot include value set "+value+" because it's actually a code system");
488      } else {
489        throw fail("Unable to find imported value set " + value);
490      }
491    }
492    ValueSetExpansionOutcome vso = new ValueSetExpanderSimple(context, allErrors).expand(vs, expParams);
493    if (vso.getError() != null) {
494      addErrors(vso.getAllErrors());
495      throw fail("Unable to expand imported value set "+vs.getUrl()+": " + vso.getError());
496    }
497    if (vs.hasVersion())
498      if (!existsInParams(exp.getParameter(), "version", new UriType(vs.getUrl() + "|" + vs.getVersion())))
499        exp.getParameter().add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(vs.getUrl() + "|" + vs.getVersion())));
500    for (Extension ex : vso.getValueset().getExpansion().getExtension()) {
501      if (ex.getUrl().equals(ToolingExtensions.EXT_EXP_TOOCOSTLY)) {
502        if (ex.getValue() instanceof BooleanType) {
503          exp.getExtension().add(new Extension(ToolingExtensions.EXT_EXP_TOOCOSTLY).setValue(new UriType(value)));
504        } else {
505          exp.getExtension().add(ex);
506        }
507      } 
508    }
509    for (ValueSetExpansionParameterComponent p : vso.getValueset().getExpansion().getParameter()) {
510      if (!existsInParams(exp.getParameter(), p.getName(), p.getValue()))
511        exp.getParameter().add(p);
512    }
513    copyExpansion(vso.getValueset().getExpansion().getContains());
514    canBeHeirarchy = false; // if we're importing a value set, we have to be combining, so we won't try for a heirarchy
515    return vso.getValueset();
516  }
517
518  public void copyExpansion(List<ValueSetExpansionContainsComponent> list) {
519    for (ValueSetExpansionContainsComponent cc : list) {
520       ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
521       n.setSystem(cc.getSystem());
522       n.setCode(cc.getCode());
523       n.setAbstract(cc.getAbstract());
524       n.setInactive(cc.getInactive());
525       n.setDisplay(cc.getDisplay());
526       n.getDesignation().addAll(cc.getDesignation());
527
528       String s = key(n);
529       if (!map.containsKey(s) && !excludeKeys.contains(s)) {
530         codes.add(n);
531         map.put(s, n);
532         total++;
533       }
534       copyExpansion(cc.getContains());
535    }
536  }
537
538  private void addErrors(List<String> errs) {
539    for (String s : errs) {
540      if (!allErrors.contains(s)) {
541        allErrors.add(s);
542      }
543    }
544  }
545
546  private void copyImportContains(List<ValueSetExpansionContainsComponent> list, ValueSetExpansionContainsComponent parent, Parameters expParams, List<ValueSet> filter) throws FHIRException {
547    for (ValueSetExpansionContainsComponent c : list) {
548      c.checkNoModifiers("Imported Expansion in Code System", "expanding");
549      ValueSetExpansionContainsComponent np = addCode(c.getSystem(), c.getCode(), c.getDisplay(), parent, null, expParams, c.getAbstract(), c.getInactive(), filter);
550      copyImportContains(c.getContains(), np, expParams, filter);
551    }
552  }
553
554  private void includeCodes(ConceptSetComponent inc, ValueSetExpansionComponent exp, Parameters expParams, boolean heirarchical, List<Extension> extensions) throws ETooCostly, FileNotFoundException, IOException, FHIRException {
555    inc.checkNoModifiers("Compose.include", "expanding");
556    List<ValueSet> imports = new ArrayList<ValueSet>();
557    for (UriType imp : inc.getValueSet()) {
558      imports.add(importValueSet(imp.getValue(), exp, expParams));
559    }
560
561    if (!inc.hasSystem()) {
562      if (imports.isEmpty()) // though this is not supposed to be the case
563        return;
564      ValueSet base = imports.get(0);
565      imports.remove(0);
566      base.checkNoModifiers("Imported ValueSet", "expanding");
567      copyImportContains(base.getExpansion().getContains(), null, expParams, imports);
568    } else {
569      CodeSystem cs = context.fetchCodeSystem(inc.getSystem());
570      if (isServerSide(inc.getSystem()) || (cs == null || (cs.getContent() != CodeSystemContentMode.COMPLETE && cs.getContent() != CodeSystemContentMode.FRAGMENT))) {
571        doServerIncludeCodes(inc, heirarchical, exp, imports, expParams, extensions);
572      } else {
573        doInternalIncludeCodes(inc, exp, expParams, imports, cs);
574      }
575    }
576  }
577
578  private void doServerIncludeCodes(ConceptSetComponent inc, boolean heirarchical, ValueSetExpansionComponent exp, List<ValueSet> imports, Parameters expParams, List<Extension> extensions) throws FHIRException {
579    ValueSetExpansionOutcome vso = context.expandVS(inc, heirarchical);
580    if (vso.getError() != null) {
581      throw failTSE("Unable to expand imported value set: " + vso.getError());
582    }
583    ValueSet vs = vso.getValueset();
584    if (vs.hasVersion()) {
585      if (!existsInParams(exp.getParameter(), "version", new UriType(vs.getUrl() + "|" + vs.getVersion()))) {
586        exp.getParameter().add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(vs.getUrl() + "|" + vs.getVersion())));
587      }
588    }
589    for (ValueSetExpansionParameterComponent p : vso.getValueset().getExpansion().getParameter()) {
590      if (!existsInParams(exp.getParameter(), p.getName(), p.getValue())) {
591        exp.getParameter().add(p);
592      }
593    }
594    for (Extension ex : vs.getExpansion().getExtension()) {
595      if (Utilities.existsInList(ex.getUrl(), ToolingExtensions.EXT_EXP_TOOCOSTLY, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed")) {
596        if (!hasExtension(extensions, ex.getUrl())) {
597          extensions.add(ex);
598        }
599      }
600    }
601    for (ValueSetExpansionContainsComponent cc : vs.getExpansion().getContains()) {
602      addCodeAndDescendents(cc, null, expParams, imports);
603    }
604  }
605
606  private boolean hasExtension(List<Extension> extensions, String url) {
607    for (Extension ex : extensions) {
608      if (ex.getUrl().equals(url)) {
609        return true;
610      }
611    }
612    return false;
613  }
614
615  public void doInternalIncludeCodes(ConceptSetComponent inc, ValueSetExpansionComponent exp, Parameters expParams, List<ValueSet> imports, CodeSystem cs) throws NoTerminologyServiceException, TerminologyServiceException, FHIRException {
616    if (cs == null) {
617      if (context.isNoTerminologyServer())
618        throw failTSE("Unable to find code system " + inc.getSystem().toString());
619      else
620        throw failTSE("Unable to find code system " + inc.getSystem().toString());
621    }
622    cs.checkNoModifiers("Code System", "expanding");
623    if (cs.getContent() != CodeSystemContentMode.COMPLETE && cs.getContent() != CodeSystemContentMode.FRAGMENT)
624      throw failTSE("Code system " + inc.getSystem().toString() + " is incomplete");
625    if (cs.hasVersion())
626      if (!existsInParams(exp.getParameter(), "version", new UriType(cs.getUrl() + "|" + cs.getVersion())))
627        exp.getParameter().add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(cs.getUrl() + "|" + cs.getVersion())));
628
629    if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) {
630      // special case - add all the code system
631      for (ConceptDefinitionComponent def : cs.getConcept()) {
632        addCodeAndDescendents(cs, inc.getSystem(), def, null, expParams, imports, null, new AllConceptsFilter());
633      }
634      if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
635        addFragmentWarning(exp, cs);
636      }
637      if (cs.getContent() == CodeSystemContentMode.EXAMPLE) {
638        addExampleWarning(exp, cs);
639      }      
640    }
641
642    if (!inc.getConcept().isEmpty()) {
643      canBeHeirarchy = false;
644      for (ConceptReferenceComponent c : inc.getConcept()) {
645        c.checkNoModifiers("Code in Code System", "expanding");
646        ConceptDefinitionComponent def = CodeSystemUtilities.findCode(cs.getConcept(), c.getCode());
647        Boolean inactive = false; // default is true if we're a fragment and  
648        if (def == null) {
649          if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
650            addFragmentWarning(exp, cs);
651          } else if (cs.getContent() == CodeSystemContentMode.EXAMPLE) {
652              addExampleWarning(exp, cs);
653          } else {
654            if (checkCodesWhenExpanding) {
655              throw failTSE("Unable to find code '" + c.getCode() + "' in code system " + cs.getUrl());
656            }
657          }
658        } else {
659          inactive = CodeSystemUtilities.isInactive(cs, def);
660        }
661        addCode(inc.getSystem(), c.getCode(), !Utilities.noString(c.getDisplay()) ? c.getDisplay() : def == null ? null : def.getDisplay(), null, convertDesignations(c.getDesignation()), expParams, false, inactive, imports);
662      }
663    }
664    if (inc.getFilter().size() > 1) {
665      canBeHeirarchy = false; // which will bt the case if we get around to supporting this
666      throw failTSE("Multiple filters not handled yet"); // need to and them, and this isn't done yet. But this shouldn't arise in non loinc and snomed value sets
667    }
668    if (inc.getFilter().size() == 1) {
669      if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
670        addFragmentWarning(exp, cs);
671      }
672      ConceptSetFilterComponent fc = inc.getFilter().get(0);
673      if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.ISA) {
674        // special: all codes in the target code system under the value
675        ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
676        if (def == null)
677          throw failTSE("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
678        addCodeAndDescendents(cs, inc.getSystem(), def, null, expParams, imports, null, new AllConceptsFilter());
679      } else if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.ISNOTA) {
680        // special: all codes in the target code system that are not under the value
681        ConceptDefinitionComponent defEx = getConceptForCode(cs.getConcept(), fc.getValue());
682        if (defEx == null)
683          throw failTSE("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
684        for (ConceptDefinitionComponent def : cs.getConcept()) {
685          addCodeAndDescendents(cs, inc.getSystem(), def, null, expParams, imports, defEx, new AllConceptsFilter());
686        }
687      } else if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.DESCENDENTOF) {
688        // special: all codes in the target code system under the value
689        ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
690        if (def == null)
691          throw failTSE("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
692        for (ConceptDefinitionComponent c : def.getConcept())
693          addCodeAndDescendents(cs, inc.getSystem(), c, null, expParams, imports, null, new AllConceptsFilter());
694        if (def.hasUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK)) {
695          List<ConceptDefinitionComponent> children = (List<ConceptDefinitionComponent>) def.getUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK);
696          for (ConceptDefinitionComponent c : children)
697            addCodeAndDescendents(cs, inc.getSystem(), c, null, expParams, imports, null, new AllConceptsFilter());
698        }
699
700      } else if ("display".equals(fc.getProperty()) && fc.getOp() == FilterOperator.EQUAL) {
701        // gg; note: wtf is this: if the filter is display=v, look up the code 'v', and see if it's diplsay is 'v'?
702        canBeHeirarchy = false;
703        ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
704        if (def != null) {
705          if (isNotBlank(def.getDisplay()) && isNotBlank(fc.getValue())) {
706            if (def.getDisplay().contains(fc.getValue())) {
707              addCode(inc.getSystem(), def.getCode(), def.getDisplay(), null, def.getDesignation(), expParams, CodeSystemUtilities.isNotSelectable(cs, def), CodeSystemUtilities.isInactive(cs, def),
708                  imports);
709            }
710          }
711        }
712      } else if (isDefinedProperty(cs, fc.getProperty())) {
713        for (ConceptDefinitionComponent def : cs.getConcept()) {
714          addCodeAndDescendents(cs, inc.getSystem(), def, null, expParams, imports, null, new PropertyFilter(fc, getPropertyDefinition(cs, fc.getProperty())));
715        }
716      } else {
717        throw fail("Search by property[" + fc.getProperty() + "] and op[" + fc.getOp() + "] is not supported yet");
718      }
719    }
720  }
721
722  private PropertyComponent getPropertyDefinition(CodeSystem cs, String property) {
723    for (PropertyComponent cp : cs.getProperty()) {
724      if (cp.getCode().equals(property)) {
725        return cp;
726      }
727    }
728    return null;
729  }
730
731  private boolean isDefinedProperty(CodeSystem cs, String property) {
732    for (PropertyComponent cp : cs.getProperty()) {
733      if (cp.getCode().equals(property)) {
734        return true;
735      }
736    }
737    return false;
738  }
739
740  private void addFragmentWarning(ValueSetExpansionComponent exp, CodeSystem cs) {
741    String url = cs.getVersionedUrl();
742    for (ValueSetExpansionParameterComponent p : exp.getParameter()) {
743      if ("fragment".equals(p.getName()) && p.hasValueUriType() && url.equals(p.getValue().primitiveValue())) { 
744        return;
745      }     
746    }
747    exp.addParameter().setName("fragment").setValue(new UriType(url));
748  }
749
750  private void addExampleWarning(ValueSetExpansionComponent exp, CodeSystem cs) {
751    String url = cs.getVersionedUrl();
752    for (ValueSetExpansionParameterComponent p : exp.getParameter()) {
753      if ("example".equals(p.getName()) && p.hasValueUriType() && url.equals(p.getValue().primitiveValue())) { 
754        return;
755      }     
756    }
757    exp.addParameter().setName("example").setValue(new UriType(url));
758  }
759  
760  private List<ConceptDefinitionDesignationComponent> convertDesignations(List<ConceptReferenceDesignationComponent> list) {
761    List<ConceptDefinitionDesignationComponent> res = new ArrayList<CodeSystem.ConceptDefinitionDesignationComponent>();
762    for (ConceptReferenceDesignationComponent t : list) {
763      ConceptDefinitionDesignationComponent c = new ConceptDefinitionDesignationComponent();
764      c.setLanguage(t.getLanguage());
765      c.setUse(t.getUse());
766      c.setValue(t.getValue());
767    }
768    return res;
769  }
770
771  private String key(String uri, String code) {
772    return "{" + uri + "}" + code;
773  }
774
775  private String key(ValueSetExpansionContainsComponent c) {
776    return key(c.getSystem(), c.getCode());
777  }
778
779  private FHIRException fail(String msg) {
780    allErrors.add(msg);
781    return new FHIRException(msg);
782  }
783
784  private ETooCostly failCostly(String msg) {
785    allErrors.add(msg);
786    return new ETooCostly(msg);
787  }
788
789  private TerminologyServiceException failTSE(String msg) {
790    allErrors.add(msg);
791    return new TerminologyServiceException(msg);
792  }
793
794  public Collection<? extends String> getAllErrors() {
795    return allErrors;
796  }
797
798  public boolean isCheckCodesWhenExpanding() {
799    return checkCodesWhenExpanding;
800  }
801
802  public void setCheckCodesWhenExpanding(boolean checkCodesWhenExpanding) {
803    this.checkCodesWhenExpanding = checkCodesWhenExpanding;
804  }
805
806}