diff --git a/modeling/ontology-mapping/src/main/java/com/ucuenca/pentaho/plugin/step/r2rml/R2RMLGenerator.java b/modeling/ontology-mapping/src/main/java/com/ucuenca/pentaho/plugin/step/r2rml/R2RMLGenerator.java index 61a3a8d..6cd8d56 100644 --- a/modeling/ontology-mapping/src/main/java/com/ucuenca/pentaho/plugin/step/r2rml/R2RMLGenerator.java +++ b/modeling/ontology-mapping/src/main/java/com/ucuenca/pentaho/plugin/step/r2rml/R2RMLGenerator.java @@ -92,7 +92,7 @@ public void process() throws KettleException { while (rsClass.next()) { ClassificationBean classBean = new ClassificationBean(rsClass); String relativeURI = classBean.getRelativeUri() != null && classBean.getRelativeUri().length() > 0 - && classBean.getRelativeUri().matches("([a-z0-9]*)(\\/|#)") ? classBean.getRelativeUri() : ""; + && classBean.getRelativeUri().matches("(#)?([a-z0-9]*)(\\/|#)") ? classBean.getRelativeUri() : ""; String id = classBean.getId(); String entityId = classBean.getUriFieldId(); String entityRealId = ""; @@ -126,7 +126,7 @@ public void process() throws KettleException { ) .addProperty(RR.subjectMap, r2rmlModel.createResource() - .addProperty(RR.template, this.baseURI + relativeURI + "{" + id + "_ID}") + .addProperty((relativeURI.compareTo("#external#")==0)?RR.column:RR.template, (relativeURI.compareTo("#external#")==0)?"" + id + "_ID":this.baseURI + relativeURI + "{" + id + "_ID}") .addProperty(RR.cclass, /*this.getOntologyURI(vocPrefix) +*/ ResourceFactory.createProperty(prefixes.get(vocPrefix)[0], vocEntity.split(prefixes.get(vocPrefix)[0])[1]) diff --git a/pom.xml b/pom.xml index 5a14020..f656dd7 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ - ${project.build.directory} + /ExtBiblioDigital/Pentaho/data-integration ${project.version} 5.1.preview.506 2.12.0 diff --git a/pom.xml~ b/pom.xml~ new file mode 100644 index 0000000..5a14020 --- /dev/null +++ b/pom.xml~ @@ -0,0 +1,43 @@ + + + 4.0.0 + com.ucuenca + ucuenca + 0.0.1-SNAPSHOT + LOD Platform UCUENCA + pom + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.7 + 1.7 + + + + + + ${project.build.directory} + ${project.version} + 5.1.preview.506 + 2.12.0 + 1.4.184 + 1.0.3-SNAPSHOT + 2.6.10 + 1.0.b2 + 3.7 + 2.6 + + + no-released-libs + tools + specification + modeling + generation + publication + exploitation + + diff --git a/publication/FusekiLoader/src/main/resources/plugin/fuseki/fuseki-server.jar b/publication/FusekiLoader/src/main/resources/plugin/fuseki/fuseki-server.jar deleted file mode 100644 index 3c7ee5a..0000000 Binary files a/publication/FusekiLoader/src/main/resources/plugin/fuseki/fuseki-server.jar and /dev/null differ diff --git a/specification/marc21Loader/src/main/java/com/ucuenca/pentaho/plugin/step/marc21loader/MARC21Loader.java b/specification/marc21Loader/src/main/java/com/ucuenca/pentaho/plugin/step/marc21loader/MARC21Loader.java index 49fdc3c..06d4e8c 100755 --- a/specification/marc21Loader/src/main/java/com/ucuenca/pentaho/plugin/step/marc21loader/MARC21Loader.java +++ b/specification/marc21Loader/src/main/java/com/ucuenca/pentaho/plugin/step/marc21loader/MARC21Loader.java @@ -1,14 +1,14 @@ package com.ucuenca.pentaho.plugin.step.marc21loader; - + import info.freelibrary.marc4j.converter.impl.AnselToUnicode; import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; +import java.util.Iterator; +import java.util.List; import javax.xml.transform.Result; import javax.xml.transform.Source; @@ -20,32 +20,32 @@ import javax.xml.transform.stream.StreamSource; import org.marc4j.MarcDirStreamReader; -import org.marc4j.MarcStreamReader; +import org.marc4j.MarcStreamReader; import org.marc4j.MarcStreamWriter; import org.marc4j.MarcXmlWriter; -import org.marc4j.marc.ControlField; -import org.marc4j.marc.DataField; -import org.marc4j.marc.Leader; -import org.marc4j.marc.Record; +import org.marc4j.marc.ControlField; +import org.marc4j.marc.DataField; +import org.marc4j.marc.Leader; +import org.marc4j.marc.Record; import org.marc4j.marc.Subfield; import org.marc4j.util.MarcXmlDriver; -import org.pentaho.di.core.Const; -import org.pentaho.di.core.exception.KettleStepException; -import org.pentaho.di.core.row.RowDataUtil; -import org.pentaho.di.core.row.RowMeta; -import org.pentaho.di.trans.Trans; -import org.pentaho.di.trans.TransMeta; -import org.pentaho.di.trans.step.BaseStep; -import org.pentaho.di.trans.step.StepDataInterface; -import org.pentaho.di.trans.step.StepInterface; -import org.pentaho.di.trans.step.StepMeta; -import org.pentaho.di.trans.step.StepMetaInterface; +import org.pentaho.di.core.Const; +import org.pentaho.di.core.exception.KettleStepException; +import org.pentaho.di.core.row.RowDataUtil; +import org.pentaho.di.core.row.RowMeta; +import org.pentaho.di.trans.Trans; +import org.pentaho.di.trans.TransMeta; +import org.pentaho.di.trans.step.BaseStep; +import org.pentaho.di.trans.step.StepDataInterface; +import org.pentaho.di.trans.step.StepInterface; +import org.pentaho.di.trans.step.StepMeta; +import org.pentaho.di.trans.step.StepMetaInterface; import org.w3c.dom.Document; import org.w3c.dom.Element; import com.ucuenca.pentaho.plugin.step.marc21loader.util.MARC; import com.ucuenca.pentaho.plugin.step.marc21loader.util.MARC21; - + /** * This class is part of the demo step plug-in implementation. * It demonstrates the basics of developing a plug-in step for PDI. @@ -68,69 +68,69 @@ public class MARC21Loader extends BaseStep implements StepInterface { private MARC21LoaderMeta meta; - private MARC21LoaderData data; + private MARC21LoaderData data; private String mfn; DOMResult source; Result result; //private MarcStreamWriter writer; - - /** - * The constructor should simply pass on its arguments to the parent class. - * - * @param s step description - * @param stepDataInterface step data class - * @param c step copy - * @param t transformation description - * @param dis transformation executing + + /** + * The constructor should simply pass on its arguments to the parent class. + * + * @param s step description + * @param stepDataInterface step data class + * @param c step copy + * @param t transformation description + * @param dis transformation executing */ public MARC21Loader(StepMeta s, StepDataInterface stepDataInterface, int c, TransMeta t, Trans dis) { super(s,stepDataInterface,c,t,dis); - } - - /** - * This method is called by PDI during transformation startup. - * - * It should initialize required for step execution. - * - * The meta and data implementations passed in can safely be cast - * to the step's respective implementations. - * - * It is mandatory that super.init() is called to ensure correct behavior. - * - * Typical tasks executed here are establishing the connection to a database, - * as wall as obtaining resources, like file handles. - * - * @param smi step meta interface implementation, containing the step settings - * @param sdi step data interface implementation, used to store runtime information - * - * @return true if initialization completed successfully, false if there was an error preventing the step from working. - * - */ - public boolean init(StepMetaInterface smi, StepDataInterface sdi) - { - meta = (MARC21LoaderMeta)smi; + } + + /** + * This method is called by PDI during transformation startup. + * + * It should initialize required for step execution. + * + * The meta and data implementations passed in can safely be cast + * to the step's respective implementations. + * + * It is mandatory that super.init() is called to ensure correct behavior. + * + * Typical tasks executed here are establishing the connection to a database, + * as wall as obtaining resources, like file handles. + * + * @param smi step meta interface implementation, containing the step settings + * @param sdi step data interface implementation, used to store runtime information + * + * @return true if initialization completed successfully, false if there was an error preventing the step from working. + * + */ + public boolean init(StepMetaInterface smi, StepDataInterface sdi) + { + meta = (MARC21LoaderMeta)smi; data = (MARC21LoaderData)sdi; - - - if (super.init(meta, data)) - { - if (meta.getmarcFilename()==null || - meta.getmarcFields()==null || - meta.getmarcFilename().length()==0 || - meta.getmarcFields().length()==0 - ) - { - logError("We need both a path file and a fields to search."); - return false; + + + if (super.init(meta, data)) + { + if (meta.getmarcFilename()==null || + meta.getmarcFields()==null || + meta.getmarcFilename().length()==0 || + meta.getmarcFields().length()==0 + ) + { + logError("We need both a path file and a fields to search."); + return false; } //marcfiles validation and listing - meta.marcFilesValidator(); - - - //Lectura del archivo MARC - FileInputStream in=null; + meta.marcFilesValidator(); + + + //Lectura del archivo MARC + FileInputStream in=null; try { for(File file:meta.getMarcfiles()) { logBasic("File to be processed: "+ file.getAbsoluteFile()); @@ -154,52 +154,52 @@ public boolean init(StepMetaInterface smi, StepDataInterface sdi) //data.marcXmlOutfile.setUnicodeNormalization(true); //this.writer = new MarcStreamWriter(new FileOutputStream(out + "marcReg.mrc")); - } - - - } catch (FileNotFoundException e1) { - // TODO Auto-generated catch block - logError("FileNotFoundException " + e1); - return false; - }/*finally { - try { - in.close(); - - } catch (IOException ex) { - logError("IOException " + ex); - //Logger.getLogger(PruebaMarc21.class.getName()).log(Level.SEVERE, null, ex); - } - }*/ + } + + + } catch (FileNotFoundException e1) { + // TODO Auto-generated catch block + logError("FileNotFoundException " + e1); + return false; + }/*finally { + try { + in.close(); + + } catch (IOException ex) { + logError("IOException " + ex); + //Logger.getLogger(PruebaMarc21.class.getName()).log(Level.SEVERE, null, ex); + } + }*/ data.marcfilesIterator = data.marcfiles.iterator(); - data.processingMarcfile = data.marcfilesIterator.next(); - } - return true; + data.processingMarcfile = data.marcfilesIterator.next(); + } + return true; } - - /** - * Once the transformation starts executing, the processRow() method is called repeatedly - * by PDI for as long as it returns true. To indicate that a step has finished processing rows - * this method must call setOutputDone() and return false; - * - * Steps which process incoming rows typically call getRow() to read a single row from the - * input stream, change or add row content, call putRow() to pass the changed row on - * and return true. If getRow() returns null, no more rows are expected to come in, - * and the processRow() implementation calls setOutputDone() and returns false to - * indicate that it is done too. - * - * Steps which generate rows typically construct a new row Object[] using a call to - * RowDataUtil.allocateRowData(numberOfFields), add row content, and call putRow() to - * pass the new row on. Above process may happen in a loop to generate multiple rows, - * at the end of which processRow() would call setOutputDone() and return false; - * - * @param smi the step meta interface containing the step settings - * @param sdi the step data interface that should be used to store - * - * @return true to indicate that the function should be called again, false if the step is done + + /** + * Once the transformation starts executing, the processRow() method is called repeatedly + * by PDI for as long as it returns true. To indicate that a step has finished processing rows + * this method must call setOutputDone() and return false; + * + * Steps which process incoming rows typically call getRow() to read a single row from the + * input stream, change or add row content, call putRow() to pass the changed row on + * and return true. If getRow() returns null, no more rows are expected to come in, + * and the processRow() implementation calls setOutputDone() and returns false to + * indicate that it is done too. + * + * Steps which generate rows typically construct a new row Object[] using a call to + * RowDataUtil.allocateRowData(numberOfFields), add row content, and call putRow() to + * pass the new row on. Above process may happen in a loop to generate multiple rows, + * at the end of which processRow() would call setOutputDone() and return false; + * + * @param smi the step meta interface containing the step settings + * @param sdi the step data interface that should be used to store + * + * @return true to indicate that the function should be called again, false if the step is done */ - public synchronized boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleStepException - { - meta = (MARC21LoaderMeta)smi; + public synchronized boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleStepException + { + meta = (MARC21LoaderMeta)smi; data = (MARC21LoaderData)sdi; boolean retval=true; @@ -207,40 +207,40 @@ public synchronized boolean processRow(StepMetaInterface smi, StepDataInterface data.processingMarcfile = !data.processingMarcfile.hasNext() && data.marcfilesIterator.hasNext() ? data.marcfilesIterator.next():data.processingMarcfile; - - if (!data.processingMarcfile.hasNext() && !data.marcfilesIterator.hasNext()) + + if (!data.processingMarcfile.hasNext() && !data.marcfilesIterator.hasNext()) { if(meta.getGenMARCXML()) { logBasic("MARCXML file generated with name: " + meta.getMarcxmlFilename()); //writer.close(); data.marcXmlOutfile.close(); - } - setOutputDone(); - return false; - } - - if (first) - { - first=false; - - // the size of the incoming rows - //data.inputSize = getInputRowMeta().size(); - - // determine output field structure - data.outputRowMeta = new RowMeta();//(RowMetaInterface) getInputRowMeta().clone(); - meta.getFields(data.outputRowMeta, getStepname(), null, null, this); - - } - - // building new row - Object[] outputRow = RowDataUtil.allocateRowData(data.outputRowMeta.size()); - Integer outputIndex = 0; - + } + setOutputDone(); + return false; + } + + if (first) + { + first=false; + + // the size of the incoming rows + //data.inputSize = getInputRowMeta().size(); + + // determine output field structure + data.outputRowMeta = new RowMeta();//(RowMetaInterface) getInputRowMeta().clone(); + meta.getFields(data.outputRowMeta, getStepname(), null, null, this); + + } + + // building new row + Object[] outputRow = RowDataUtil.allocateRowData(data.outputRowMeta.size()); + Integer outputIndex = 0; + Record record = data.processingMarcfile.next(); if(meta.getGenMARCXML()) { //writer.write(record); data.marcXmlOutfile.write(record); - } + } this.getBasicControlFields(record, outputIndex); if(mfn.equals("000002") && log.isDebug()) { @@ -253,35 +253,35 @@ public synchronized boolean processRow(StepMetaInterface smi, StepDataInterface return false; } - //MARC fields reading - - String leer_campos = meta.getmarcFields(); - String[] campos = leer_campos.split("@"); - MARC21 obj_marc = new MARC21(); + //MARC fields reading + + String leer_campos = meta.getmarcFields(); + String[] campos = leer_campos.split("@"); + MARC21 obj_marc = new MARC21(); int mfnInt=Integer.parseInt(mfn); String currentField = ""; - String subfields = ""; - for (int i = 0; i < campos.length; i++) { - outputRow = RowDataUtil.allocateRowData(data.outputRowMeta.size()); + String subfields = ""; + for (int i = 0; i < campos.length; i++) { + outputRow = RowDataUtil.allocateRowData(data.outputRowMeta.size()); if (campos[i].length() == 4) { subfields = !currentField.equals(campos[i].substring(0, 3)) ? "":subfields; currentField = campos[i].substring(0, 3); subfields += campos[i].charAt(3); - - obj_marc=this.obtainField(record, mfnInt, currentField, subfields, campos[i].charAt(3)); - outputIndex=0; - outputRow[outputIndex++] = mfn; + + obj_marc=this.obtainField(record, mfnInt, currentField, subfields, campos[i].charAt(3)); + outputIndex=0; + outputRow[outputIndex++] = mfn; outputRow[outputIndex++] = new Long(obj_marc.getField()); - outputRow[outputIndex++] = obj_marc.getIndicators(); - outputRow[outputIndex++] = obj_marc.getLeadersubfields()+""; - outputRow[outputIndex++] = obj_marc.getValue()+""; + outputRow[outputIndex++] = obj_marc.getIndicators(); + outputRow[outputIndex++] = obj_marc.getLeadersubfields()+""; + outputRow[outputIndex++] = obj_marc.getValue()+""; putRow(data.outputRowMeta, outputRow); - logRowlevel("Processed field: " + campos[i]); + logRowlevel("Processed field: " + campos[i]); } else { logBasic("Field {0} is not well formatted. Omitting", campos[i]); - } - } - + } + } + //if ((linesInput%Const.ROWS_UPDATE)==0) logBasic("linenr "+linesInput); return retval; @@ -404,39 +404,39 @@ private MARC21 obtainField(Record record, int mfn, String field, String leaderSu } objMarc=new MARC21(mfn, field, indicators, subfield, leaderSubfields, data); return objMarc; - } - - /** - * This method is called by PDI once the step is done processing. - * - * The dispose() method is the counterpart to init() and should release any resources - * acquired for step execution like file handles or database connections. - * - * The meta and data implementations passed in can safely be cast - * to the step's respective implementations. - * - * It is mandatory that super.dispose() is called to ensure correct behavior. - * - * @param smi step meta interface implementation, containing the step settings - * @param sdi step data interface implementation, used to store runtime information - */ - public void dispose(StepMetaInterface smi, StepDataInterface sdi) - { - meta = (MARC21LoaderMeta)smi; - data = (MARC21LoaderData)sdi; - - super.dispose(smi, sdi); + } + + /** + * This method is called by PDI once the step is done processing. + * + * The dispose() method is the counterpart to init() and should release any resources + * acquired for step execution like file handles or database connections. + * + * The meta and data implementations passed in can safely be cast + * to the step's respective implementations. + * + * It is mandatory that super.dispose() is called to ensure correct behavior. + * + * @param smi step meta interface implementation, containing the step settings + * @param sdi step data interface implementation, used to store runtime information + */ + public void dispose(StepMetaInterface smi, StepDataInterface sdi) + { + meta = (MARC21LoaderMeta)smi; + data = (MARC21LoaderData)sdi; + + super.dispose(smi, sdi); } // // Run is were the action happens! // - // - /* + // + public void run() { logBasic("Starting to run..."); - + try { while (processRow(meta, data) && !isStopped()); @@ -446,12 +446,12 @@ public void run() logError("Unexpected error", e); setErrors(1); stopAll(); - } - finally - { - dispose(meta, data); - logBasic("Finished, processed "+linesInput+" rows, written "+linesWritten+" lines."); - markStop(); } - }*/ + finally + { + dispose(meta, data); + logBasic("Finished, processed "+linesInput+" rows, written "+linesWritten+" lines."); + markStop(); + } + } }