|
/** |
|
* Author: Pierre Lindenbaum PhD |
|
* plindenbaum@yahoo.fr |
|
* Date: 2012-11 |
|
* Motivation: RDFGraph from openoffice calc files |
|
* |
|
*/ |
|
package oocalc; |
|
|
|
import java.io.File; |
|
import java.io.FileNotFoundException; |
|
import java.io.FileReader; |
|
import java.io.InputStream; |
|
import java.util.ArrayList; |
|
import java.util.LinkedList; |
|
import java.util.List; |
|
import java.util.zip.ZipEntry; |
|
import java.util.zip.ZipFile; |
|
|
|
import javax.xml.namespace.QName; |
|
import javax.xml.stream.XMLEventReader; |
|
import javax.xml.stream.XMLInputFactory; |
|
import javax.xml.stream.XMLStreamException; |
|
import javax.xml.stream.events.Attribute; |
|
import javax.xml.stream.events.EndElement; |
|
import javax.xml.stream.events.StartElement; |
|
import javax.xml.stream.events.XMLEvent; |
|
import com.hp.hpl.jena.assembler.assemblers.AssemblerBase; |
|
import com.hp.hpl.jena.assembler.Assembler; |
|
import com.hp.hpl.jena.sparql.core.assembler.AssemblerUtils; |
|
import com.hp.hpl.jena.assembler.Mode; |
|
import com.hp.hpl.jena.datatypes.RDFDatatype; |
|
import com.hp.hpl.jena.datatypes.xsd.XSDDatatype; |
|
import com.hp.hpl.jena.rdf.model.Resource; |
|
import com.hp.hpl.jena.rdf.model.Statement; |
|
import com.hp.hpl.jena.rdf.model.StmtIterator; |
|
import com.hp.hpl.jena.rdf.model.Property; |
|
import com.hp.hpl.jena.graph.Node; |
|
import com.hp.hpl.jena.graph.Triple; |
|
import com.hp.hpl.jena.graph.TripleMatch; |
|
import com.hp.hpl.jena.graph.TripleMatchIterator; |
|
import com.hp.hpl.jena.graph.impl.GraphBase; |
|
import com.hp.hpl.jena.rdf.model.AnonId; |
|
import com.hp.hpl.jena.rdf.model.ResourceFactory; |
|
import com.hp.hpl.jena.rdf.model.impl.ModelCom; |
|
import com.hp.hpl.jena.util.iterator.ExtendedIterator; |
|
import com.hp.hpl.jena.util.iterator.NiceIterator; |
|
import com.hp.hpl.jena.sparql.core.DatasetImpl; |
|
import com.hp.hpl.jena.vocabulary.DC; |
|
import com.hp.hpl.jena.vocabulary.RDF; |
|
import com.hp.hpl.jena.vocabulary.XSD; |
|
import com.hp.hpl.jena.query.Dataset; |
|
import org.slf4j.LoggerFactory; |
|
import com.hp.hpl.jena.query.*; |
|
|
|
/** |
|
* implementation of a RDF Graph for OpenOffice calc |
|
* |
|
*/ |
|
|
|
public class OpenOfficeCalcGraph |
|
extends GraphBase |
|
{ |
|
/** logger */ |
|
protected static final org.slf4j.Logger LOG= LoggerFactory.getLogger("ooffice2rdf"); |
|
/** namespaces */ |
|
private static final String OFFICE="urn:oasis:names:tc:opendocument:xmlns:office:1.0"; |
|
private static final String TABLE="urn:oasis:names:tc:opendocument:xmlns:table:1.0"; |
|
private static final String TEXT="urn:oasis:names:tc:opendocument:xmlns:text:1.0"; |
|
private static final String NS="http://rdf.lindenb.org/"; |
|
/** attributes */ |
|
private static final QName number_columns_repeated=new QName(TABLE,"number-columns-repeated","table"); |
|
private static final QName number_rows_repeated=new QName(TABLE,"number-rows-repeated","table"); |
|
private static final QName value_type=new QName(OFFICE,"value-type","office"); |
|
private static final QName value=new QName(OFFICE,"value","office"); |
|
private static final QName name=new QName(TABLE,"name","table"); |
|
//rdf:type Node |
|
private static final Node rdfType=Node.createURI(RDF.type.getURI()); |
|
//all open office files |
|
private List<File> caclFiles=null; |
|
|
|
|
|
|
|
/** static Assembler for OpenOfficeCalcGraph |
|
* An assembler creates a Dataset(graph) from a RDF-based configuration file. |
|
* It is called by Fuseki |
|
*/ |
|
public static OpenOfficeAssembler assembler = new OpenOfficeAssembler(); |
|
|
|
|
|
public static class OpenOfficeAssembler extends AssemblerBase implements Assembler |
|
{ |
|
@Override |
|
public Object open( Assembler a, Resource root, Mode mode ) |
|
{ |
|
//read the configuration an get the files |
|
List<File> files=new ArrayList<File>(); |
|
StmtIterator iter=root.listProperties(fileRsrc); |
|
while(iter.hasNext()) |
|
{ |
|
Statement stmt=iter.nextStatement(); |
|
if(!stmt.getObject().isLiteral()) throw new RuntimeException("Not a literal "+stmt); |
|
String lit=stmt.getString(); |
|
File file=new File(lit); |
|
if(!file.exists()) throw new RuntimeException("File not found : "+file); |
|
if(!file.getName().endsWith(".ods")) throw new RuntimeException("Not an .ods file : "+file); |
|
files.add(file); |
|
} |
|
iter.close(); |
|
OpenOfficeCalcGraph g=new OpenOfficeCalcGraph(files); |
|
OpenOfficeCalcModel m=new OpenOfficeCalcModel(g); |
|
Dataset ds=new DatasetImpl(m); |
|
return ds; |
|
} |
|
} |
|
|
|
/** Initializer for FUZEKI */ |
|
private static boolean init_called = false ; |
|
private static final Resource buildRsrc=ResourceFactory.createResource(NS+"build"); |
|
private static final Property fileRsrc=ResourceFactory.createProperty(NS+"file"); |
|
|
|
/** static initializer, when this class is invoked, |
|
* it tells Fuzeki that there is another assembler using Assembler.general |
|
* the resource-name for this assembler is this.buildRsrc |
|
*/ |
|
static { init() ; } |
|
private static void init() |
|
{ |
|
if(init_called) return; |
|
LOG.info("Calling OpenOfficeCalcGraph init"); |
|
AssemblerUtils.init(); |
|
Assembler.general.implementWith(buildRsrc,assembler); |
|
|
|
init_called=true; |
|
} |
|
|
|
|
|
/** RDF Model for OpenOfficeCalcGraph */ |
|
public static class OpenOfficeCalcModel extends ModelCom |
|
{ |
|
public OpenOfficeCalcModel(OpenOfficeCalcGraph g) |
|
{ |
|
super(g); |
|
} |
|
} |
|
/* one row in the spredsheet */ |
|
private static class Row |
|
{ |
|
int repeat=1; |
|
private List<Cell> cells=new ArrayList<Cell>(); |
|
} |
|
|
|
/* one cell in the spredsheet */ |
|
private static class Cell |
|
{ |
|
int repeat=1; |
|
String type=null; |
|
String value=null; |
|
String literal=null; |
|
} |
|
|
|
/** Constructor from an array of OO files */ |
|
public OpenOfficeCalcGraph(List<File> calcFiles) |
|
{ |
|
this.caclFiles=new ArrayList<File>(calcFiles); |
|
this.getPrefixMapping().setNsPrefix("office", NS); |
|
this.getPrefixMapping().setNsPrefix("xsd", XSD.getURI()); |
|
this.getPrefixMapping().setNsPrefix("dc", DC.getURI()); |
|
} |
|
|
|
|
|
@Override |
|
protected ExtendedIterator<Triple> graphBaseFind(TripleMatch matcher) |
|
{ |
|
return new TripleMatchIterator((Triple)matcher, new CellIterator()); |
|
} |
|
|
|
/** parse the openoffice files and get the Triples */ |
|
private class CellIterator extends NiceIterator<Triple> |
|
{ |
|
/** current index in array of OO files */ |
|
private int fileIndex=-1; |
|
/** buffer of triples */ |
|
private List<Triple> buffer=new LinkedList<Triple>(); |
|
/** next triple to be returned */ |
|
private Triple next=null; |
|
/** was hasNext() called ? */ |
|
private boolean hasNextCalled=false; |
|
/** current OO file opened */ |
|
private File ioFile=null; |
|
/** Zip Handler for OO file */ |
|
private ZipFile zipFile=null; |
|
/** Input Stream for current Zip entry */ |
|
private InputStream zipInputStream; |
|
/** xml-handler for current zip entry */ |
|
private XMLEventReader xmlEventReader; |
|
/* rdf subject for file */ |
|
private Node fileRsrc=null; |
|
/* rdf subject for tab */ |
|
private Node tabRsrc=null; |
|
/** current tab index */ |
|
private int tabIndex=0; |
|
/* current colun */ |
|
private int X=0; |
|
/** current row */ |
|
private int Y=0; |
|
|
|
private void add(Node s,Node p,Node o) |
|
{ |
|
this.buffer.add(Triple.create(s, p, o)); |
|
} |
|
|
|
public CellIterator() |
|
{ |
|
|
|
} |
|
|
|
private boolean isA(XMLEvent evt,String ns,String localName) |
|
{ |
|
QName q=null; |
|
|
|
if(evt.isStartElement()) |
|
{ |
|
q=evt.asStartElement().getName(); |
|
} |
|
else if(evt.isEndElement()) |
|
{ |
|
q=evt.asEndElement().getName(); |
|
} |
|
return q!=null && |
|
q.getNamespaceURI().equals(ns) && |
|
q.getLocalPart().equals(localName) |
|
; |
|
} |
|
|
|
@Override |
|
public boolean hasNext() |
|
{ |
|
if(!hasNextCalled) |
|
{ |
|
hasNextCalled=true; |
|
next=null; |
|
for(;;) |
|
{ |
|
if(!buffer.isEmpty()) |
|
{ |
|
next=buffer.remove(0); |
|
break; |
|
} |
|
|
|
try |
|
{ |
|
|
|
if(xmlEventReader==null) |
|
{ |
|
//open next file |
|
if(fileIndex+1>=OpenOfficeCalcGraph.this.caclFiles.size()) break; |
|
this.fileIndex++; |
|
this.tabIndex=0; |
|
//open XML StaX reader for current OO file |
|
XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance(); |
|
xmlInputFactory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, Boolean.TRUE); |
|
xmlInputFactory.setProperty(XMLInputFactory.IS_COALESCING, Boolean.TRUE); |
|
xmlInputFactory.setProperty(XMLInputFactory.IS_REPLACING_ENTITY_REFERENCES, Boolean.TRUE); |
|
try |
|
{ |
|
this.ioFile=OpenOfficeCalcGraph.this.caclFiles.get(this.fileIndex); |
|
this.zipFile=new ZipFile(this.ioFile); |
|
ZipEntry zipEntry=zipFile.getEntry("content.xml"); |
|
if(zipEntry==null) throw new RuntimeException("Cannot get content.xml"); |
|
this.zipInputStream=this.zipFile.getInputStream(zipEntry); |
|
xmlEventReader= xmlInputFactory.createXMLEventReader(this.zipInputStream); |
|
//describe the file as RDF |
|
this.fileRsrc=Node.createURI(this.ioFile.toURI().toASCIIString()); |
|
add(this.fileRsrc,rdfType,Node.createURI(NS+"Spreadsheet")); |
|
add(this.fileRsrc,Node.createURI(DC.title.getURI()),Node.createLiteral(this.ioFile.getName())); |
|
continue; |
|
} |
|
catch (Exception e) |
|
{ |
|
throw new RuntimeException(e); |
|
} |
|
} |
|
|
|
|
|
if(xmlEventReader.hasNext()) |
|
{ |
|
Attribute att=null; |
|
XMLEvent evt=xmlEventReader.nextEvent(); |
|
if(evt.isStartElement()) |
|
{ |
|
StartElement E=evt.asStartElement(); |
|
if(isA(E,TABLE,"table")) |
|
{ |
|
att=E.getAttributeByName(name); |
|
this.tabIndex++; |
|
//describe the tab as RDF |
|
this.tabRsrc=Node.createURI(this.ioFile.toURI().toASCIIString()+"/t"+tabIndex); |
|
add(this.tabRsrc,Node.createURI(NS+"file"),this.fileRsrc); |
|
add(this.tabRsrc,rdfType,Node.createURI(NS+"Table")); |
|
add(this.tabRsrc,Node.createURI(DC.title.getURI()),Node.createLiteral(att.getValue())); |
|
this.X=0; |
|
this.Y=0; |
|
} |
|
else if(isA(E,TABLE,"table-row")) |
|
{ |
|
//parse the row |
|
Row row=parseRow(E); |
|
//create the statements for that row |
|
for(int i=0;i< row.repeat;++i) |
|
{ |
|
this.X=0; |
|
this.Y++; |
|
for(Cell cell:row.cells) |
|
{ |
|
for(int j=0;j< cell.repeat;++j) |
|
{ |
|
this.X++; |
|
if(cell.value==null && cell.literal==null) continue; |
|
Node subject=Node.createURI(this.ioFile.toURI().toASCIIString()+"/t"+tabIndex+"/y"+Y+"/x"+X); |
|
add(subject,Node.createURI(NS+"table"),this.tabRsrc); |
|
add(subject,rdfType,Node.createURI(NS+"Cell")); |
|
|
|
add(subject,Node.createURI(NS+"X"),Node.createLiteral(String.valueOf(X),null,XSDDatatype.XSDint)); |
|
add(subject,Node.createURI(NS+"Y"),Node.createLiteral(String.valueOf(Y),null,XSDDatatype.XSDint)); |
|
Node cellValue=null; |
|
if(cell.type!=null && cell.value!=null) |
|
{ |
|
XSDDatatype dataType=XSDDatatype.XSDstring; |
|
if(cell.type.equals("float")) |
|
{ |
|
dataType=XSDDatatype.XSDfloat; |
|
} |
|
else if(cell.type.equals("int")) |
|
{ |
|
dataType=XSDDatatype.XSDint; |
|
} |
|
cellValue=Node.createLiteral(cell.value, null, dataType); |
|
} |
|
else |
|
{ |
|
cellValue=Node.createLiteral(String.valueOf(cell.literal)); |
|
} |
|
add( subject, |
|
Node.createURI(NS+"value"), |
|
cellValue |
|
); |
|
|
|
} |
|
} |
|
} |
|
} |
|
} |
|
else if(evt.isEndElement()) |
|
{ |
|
if(isA(evt,TABLE,"table")) |
|
{ |
|
this.tabRsrc=null; |
|
} |
|
} |
|
} |
|
else //we're done for that file. |
|
{ |
|
this.xmlEventReader.close(); |
|
this.zipInputStream.close(); |
|
this.zipFile.close(); |
|
this.xmlEventReader=null; |
|
this.zipInputStream=null; |
|
this.zipFile=null; |
|
this.fileRsrc=null; |
|
this.ioFile=null; |
|
} |
|
} |
|
catch(Exception err) |
|
{ |
|
throw new RuntimeException(err); |
|
} |
|
} |
|
} |
|
return next!=null; |
|
} |
|
|
|
@Override |
|
public void close() |
|
{ |
|
try { if(this.xmlEventReader!=null) this.xmlEventReader.close(); } catch (Exception e) {} |
|
this.xmlEventReader=null; |
|
try { if(this.zipInputStream!=null) this.zipInputStream.close(); } catch (Exception e) {} |
|
this.zipInputStream=null; |
|
try { if(this.zipFile!=null) this.zipFile.close(); } catch (Exception e) {} |
|
this.zipFile=null; |
|
this.buffer.clear(); |
|
this.fileIndex=caclFiles.size(); |
|
} |
|
|
|
@Override |
|
public Triple next() |
|
{ |
|
if(!hasNextCalled) hasNext(); |
|
if(!hasNext()) throw new IllegalStateException(); |
|
Triple t=next; |
|
next=null; |
|
hasNextCalled=false; |
|
return t; |
|
} |
|
|
|
|
|
/** parses a table:table-row */ |
|
private Row parseRow(StartElement root) |
|
throws XMLStreamException |
|
{ |
|
Row row=new Row(); |
|
Attribute att=root.getAttributeByName(number_rows_repeated); |
|
|
|
if(att!=null) |
|
{ |
|
row.repeat=Integer.parseInt(att.getValue()); |
|
} |
|
while(this.xmlEventReader.hasNext()) |
|
{ |
|
XMLEvent evt=this.xmlEventReader.nextEvent(); |
|
if(evt.isStartElement()) |
|
{ |
|
StartElement E=evt.asStartElement(); |
|
if(isA(E,TABLE,"table-cell")) |
|
{ |
|
row.cells.add(parseCell(E)); |
|
} |
|
} |
|
else if(evt.isEndElement()) |
|
{ |
|
if(isA(evt,TABLE,"table-row")) |
|
{ |
|
break; |
|
} |
|
} |
|
} |
|
return row; |
|
} |
|
/** parses a table:table-cell */ |
|
private Cell parseCell(StartElement root) |
|
throws XMLStreamException |
|
{ |
|
Cell cell=new Cell(); |
|
Attribute att=root.getAttributeByName(number_columns_repeated); |
|
|
|
if(att!=null) |
|
{ |
|
cell.repeat=Integer.parseInt(att.getValue()); |
|
} |
|
|
|
att=root.getAttributeByName(value_type); |
|
if(att!=null) |
|
{ |
|
cell.type=att.getValue(); |
|
} |
|
att=root.getAttributeByName(value); |
|
if(att!=null) |
|
{ |
|
cell.value=att.getValue(); |
|
cell.literal=cell.value; |
|
} |
|
|
|
|
|
while(this.xmlEventReader.hasNext()) |
|
{ |
|
XMLEvent evt=this.xmlEventReader.nextEvent(); |
|
if(evt.isStartElement()) |
|
{ |
|
StartElement E=evt.asStartElement(); |
|
if(isA(E,TEXT,"p")) |
|
{ |
|
cell.literal=parseText(E); |
|
} |
|
} |
|
else if(evt.isEndElement()) |
|
{ |
|
if(isA(evt,TABLE,"table-cell")) |
|
{ |
|
break; |
|
} |
|
} |
|
} |
|
return cell; |
|
} |
|
|
|
/** returns the content of <text:p/> */ |
|
private String parseText(StartElement root) |
|
throws XMLStreamException |
|
{ |
|
StringBuilder b=new StringBuilder(); |
|
while(xmlEventReader.hasNext()) |
|
{ |
|
XMLEvent evt=this.xmlEventReader.nextEvent(); |
|
if(evt.isStartElement()) |
|
{ |
|
throw new IllegalStateException(); |
|
} |
|
else if(evt.isEndElement()) |
|
{ |
|
if(isA(evt,TEXT,"p")) |
|
{ |
|
return b.toString(); |
|
} |
|
} |
|
else if(evt.isCharacters()) |
|
{ |
|
b.append(evt.asCharacters().getData()); |
|
} |
|
} |
|
throw new IllegalStateException(); |
|
} |
|
|
|
} |
|
public static void main(String[] args) throws Exception |
|
{ |
|
|
|
if(args.length<2) |
|
{ |
|
System.err.println("Usage: query.sparql file1.ods, file2.ods... filen.ods"); |
|
return; |
|
} |
|
|
|
List<File> files=new ArrayList<File>(); |
|
for(int optind=1;optind< args.length;++optind) |
|
{ |
|
files.add(new File(args[optind])); |
|
} |
|
OpenOfficeCalcGraph g=new OpenOfficeCalcGraph(files); |
|
OpenOfficeCalcModel m=new OpenOfficeCalcModel(g); |
|
|
|
|
|
|
|
com.hp.hpl.jena.query.Query query = QueryFactory.read(args[0]) ; |
|
LOG.info("starting query"); |
|
QueryExecution qexec = QueryExecutionFactory.create(query, m) ; |
|
try { |
|
ResultSet results = qexec.execSelect(); |
|
ResultSetFormatter.out(System.out,results,g.getPrefixMapping()); |
|
|
|
} finally { qexec.close() ; } |
|
|
|
} |
|
} |
|
|
