From ae96817c3df5e6a87116efcc483c0a7413d10acc Mon Sep 17 00:00:00 2001
From: Martin Fenner <martin.fenner@datacite.org>
Date: Thu, 27 Dec 2018 11:27:39 +0100
Subject: [PATCH] force utf-8 encoding of xml. #165

---
 app/models/concerns/crosscitable.rb |  25 ++++++++++++
 app/models/doi.rb                   |   4 +-
 spec/concerns/crosscitable_spec.rb  |  22 +++++++++++
 spec/fixtures/files/utf-16.xml      | Bin 0 -> 4362 bytes
 spec/fixtures/files/utf-8_bom.xml   |  59 ++++++++++++++++++++++++++++
 5 files changed, 108 insertions(+), 2 deletions(-)
 create mode 100644 spec/fixtures/files/utf-16.xml
 create mode 100644 spec/fixtures/files/utf-8_bom.xml

diff --git a/app/models/concerns/crosscitable.rb b/app/models/concerns/crosscitable.rb
index 962db7090..0692eb792 100644
--- a/app/models/concerns/crosscitable.rb
+++ b/app/models/concerns/crosscitable.rb
@@ -83,6 +83,31 @@ def update_xml
       write_attribute(:xml, xml)
     end
 
+    def clean_xml(string)
+      begin
+        return nil unless string.present?
+
+        # enforce utf-8
+        string = string.force_encoding("UTF-8")
+      rescue ArgumentError, Encoding::CompatibilityError => error
+        # convert utf-16 to utf-8
+        string = string.force_encoding('UTF-16').encode('UTF-8')
+        string.gsub!('encoding="UTF-16"', 'encoding="UTF-8"')
+      end
+
+      # remove optional bom
+      string.gsub!("\xEF\xBB\xBF", '')
+
+      # remove leading and trailing whitespace
+      string = string.strip
+
+      return nil unless string.start_with?('<?xml version=') || string.start_with?('<resource ')
+
+      # make sure xml is valid
+      doc = Nokogiri::XML(string) { |config| config.strict.noblanks }
+      doc.to_xml.strip
+    end
+
     def well_formed_xml(string)
       return nil unless string.present?
   
diff --git a/app/models/doi.rb b/app/models/doi.rb
index 97c9d0c7e..85646bf9d 100644
--- a/app/models/doi.rb
+++ b/app/models/doi.rb
@@ -345,7 +345,7 @@ def self.import_one(doi_id: nil)
       return nil
     end
 
-    string = doi.current_metadata.present? ? doi.from_xml(doi.current_metadata.xml.to_s.force_encoding("UTF-8")) : nil
+    string = doi.current_metadata.present? ? doi.clean_xml(doi.current_metadata.xml) : nil
     unless string.present?
       logger.error "[MySQL] No metadata for DOI " + doi.doi + " found: " + doi.current_metadata.inspect
       return nil
@@ -433,7 +433,7 @@ def self.import_by_day_missing(options={})
 
     Doi.where(xml: nil).where(created: from_date.midnight..from_date.end_of_day).find_each do |doi|
       begin
-        string = doi.current_metadata.present? ? doi.from_xml(doi.current_metadata.xml.to_s.force_encoding("UTF-8")) : nil
+        string = doi.current_metadata.present? ? doi.clean_xml(doi.current_metadata.xml) : nil
         unless string.present?
           logger.error "[MySQL] No metadata for DOI " + doi.doi + " found."
           return nil
diff --git a/spec/concerns/crosscitable_spec.rb b/spec/concerns/crosscitable_spec.rb
index 6ef5d94de..3eed45830 100644
--- a/spec/concerns/crosscitable_spec.rb
+++ b/spec/concerns/crosscitable_spec.rb
@@ -5,6 +5,28 @@
 
   subject { DoisController.new }
 
+  context "clean_xml" do
+    it "clean_xml" do
+      string = file_fixture('datacite.xml').read
+      expect(subject.from_xml(string)).to eq(string)
+    end
+
+    it "clean_xml malformed" do
+      string = file_fixture('datacite_malformed.xml').read
+      expect { subject.clean_xml(string) }.to raise_error(Nokogiri::XML::SyntaxError, "39:18: FATAL: Premature end of data in tag resource line 2")
+    end
+
+    it "clean_xml utf-8 bom" do
+      string = file_fixture('utf-8_bom.xml').read
+      expect(subject.clean_xml(string)).to start_with('<?xml version="1.0" encoding="UTF-8"?>')
+    end
+
+    it "clean_xml utf-16" do
+      string = file_fixture('utf-16.xml').read
+      expect(subject.clean_xml(string)).to start_with('<?xml version="1.0" encoding="UTF-8"?>')
+    end
+  end
+
   context "from_xml" do
     it "from_xml" do
       string = file_fixture('datacite.xml').read
diff --git a/spec/fixtures/files/utf-16.xml b/spec/fixtures/files/utf-16.xml
new file mode 100644
index 0000000000000000000000000000000000000000..1541c3ff87166eb1dc287d2f1cf1b28d372c3471
GIT binary patch
literal 4362
zcmeHKO;6iE5S=r>!s1+FXe(M!YJ^l3p+Z}gfas}mK5-!qiX9+-zU_N6-RvedAvv^{
zA}firGdnx;=5v4klBOKUgAC<HBDs^EjHQrFMv}`r*_0h=$Tof>P;%*Dy$fnC1Eroz
zOFqaJxOTyFAc<^1-Wa?EI45}P;63^yKga4m-YK85ucX9ti?tZfy*>?mzrvX#>>XhB
zB~IUl{8%pKR8Hg!60ab0C@IdRCIw`qs#{O>H2M2j_naq?*8x`<ST=upSE(tuGH~?*
z4MI7wT!S{oX+7YzC9h%Iyqu7`0jzjtozx*DkW!Tx!#1P2-3`U8tJZDc#1PLaj&9ew
zmTao`D7^^`y3iq4y!!Y@o1@y*mQ3Nnp8E5>e3oNqnZRG$KyXLi${Y1y4BvghrvcyY
zVVzzj^*g@za9&(1o6%(YbZ~x8v1D8nGfnHLZWn2F^iPmOgcJD)SyT9Ngt&hJ-cQit
z8vky<F~$Ew=u6$^Pxt#xJAY0peLNZVXR51ZQ3A{9Ps6a(=#AuPW>JKgE^$&6@D<Y5
z>>21PHJYjfMsyD89ju@0o6p!1Jk<6vExfi>n?lT*3-*Sz<#?VcdZp?`9PgkDaXrN^
zvzgz_t`uv}*fNq=$F_y2r>{%J*dcoaEE#RNVtVl$uWMm?tPz?Ak@~;mS*;Ad);1A)
z%-$PBp<lI%;3=**?Sqe9+2#cH6843!x;Mjq-?JmUk5Ca<A4i~M8t0LAAo7h-2|M_;
zY8rv34{lP}5r%j#wN|z@4>)bK^=x*q0{{9d*Z8dZu!2r>C$oq3e+o`kDyz;JcGwB-
zY)&$iOpChsy+s{Pv4?A%iH2A)-JPC9(xc+?e16rV*u?&c>P>7o->^3P+rCk!f$P7X
z7EV#$peJj3GVABG?<rm<vrvnFKnA_*v9O4mrzSv{U5^n$<j3#=XRrbf&s^jVd)*>!
zHQz1nS|)L|SX@_6iylT?HCi8+Ri$`jo3osp0K)<BOYq4RL6@zb;{3EY&AW*&f*0oJ
zd`CG_sf=Of7bnyfRvzGGUjr<1IL|PfQhdly?H`lsbI3BA(U4)E%Q$bRMNmDLt1FAU
zo$l235SeI|hm$+CUS8GC!EJXURwvVldBPi&o8_&}3=|Va;8g1kZxI&%jO<+bU7YhK
zL21T$iQ5M8<Xp;_pmbI()(c;kLN1>%g`!JK%p+9^tbO;;gtwVAsK1;zSRd&f);&&c
zw~)fy6jxkb>|@2%%n8<~Gd#iyPHsb`u8x{^yw905-tfF{dHWzfZhkf@V}`Rfk^Vb<
zA(9J3@2=Y3$LTzgTF`5p#W|agkW2KM=|MZ}u0r4Q4pr!Mf2jJ?Chlg(C?V+{yXZmM
z!-!=cBxY&Q4Gedi^RB_s22ZK}qI{Q!Pjyv`FUyK*S9Km)-Cy;6&-r0l^)fuVz%9ew
aKf)bw-P>Y)Zu<%mE0)Zlx*AbeWs=_vE%i14

literal 0
HcmV?d00001

diff --git a/spec/fixtures/files/utf-8_bom.xml b/spec/fixtures/files/utf-8_bom.xml
new file mode 100644
index 000000000..4338047c4
--- /dev/null
+++ b/spec/fixtures/files/utf-8_bom.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<resource xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://datacite.org/schema/kernel-4" xsi:schemaLocation="http://datacite.org/schema/kernel-4 http://schema.datacite.org/meta/kernel-4.1/metadata.xsd">
+<identifier identifierType="DOI">10.25664/art-0226</identifier>
+<creators>
+<creator>
+  <creatorName nameType="Personal">Fournier, Jacques</creatorName>
+  <givenName>Jacques</givenName>
+  <familyName>Fournier</familyName>
+</creator>
+<creator>
+  <creatorName nameType="Personal">Lechat, Christian</creatorName>
+  <givenName>Christian</givenName>
+  <familyName>Lechat</familyName>
+</creator>
+<creator>
+  <creatorName nameType="Personal">Courtecuisse, Régis</creatorName>
+  <givenName>Régis</givenName>
+  <familyName>Courtecuisse</familyName>
+</creator>
+</creators>
+<titles>
+  <title xml:lang="en">The genera Kretzschmariella and Nemania (Xylariaceae) in Guadeloupe and Martinique (French West Indies)</title>
+</titles>
+<publisher>Ascomycete.org</publisher>
+<publicationYear>2018</publicationYear>
+<dates>
+  <date dateType="Issued">2018-02-25</date>
+</dates>
+<language>en</language>
+<resourceType resourceTypeGeneral="Text">Journal Article</resourceType>
+<relatedIdentifiers>
+  <relatedIdentifier relatedIdentifierType="EISSN" relationType="IsPartOf">2100-0840</relatedIdentifier>
+</relatedIdentifiers>
+<alternateIdentifiers>
+  <alternateIdentifier alternateIdentifierType="Volume">10</alternateIdentifier>
+  <alternateIdentifier alternateIdentifierType="Issue">1</alternateIdentifier>
+  <alternateIdentifier alternateIdentifierType="Pages">1-47</alternateIdentifier>
+</alternateIdentifiers>
+<sizes>
+  <size>10362 kB</size>
+</sizes>
+<formats>
+  <format>PDF</format>
+</formats>
+<rightsList>
+  <rights xml:lang="fr" rightsURI="https://creativecommons.org/licenses/by-nc-nd/3.0/fr/">Creative Commons BY-NC-ND 3.0 FR</rights>
+</rightsList>
+<descriptions>
+  <description xml:lang="en" descriptionType="Abstract">This survey deals with the Nemania taxa collected in the French West Indies in the course of an ongoing inventorial work on the mycobiota of these islands initiated in 2003. Based on the evaluation and comparison of their morphological characters, sixteen taxa are described, illustrated and discussed, including seven known taxa, viz.: N. beaumontii, N. bipapillata, N. caries, N. chestersii var. microspora, N. diffusa, N. immersidiscus and N. subaenea. Nine new taxa are proposed, including N. albofarcta, N. colubrina, N. discostoma, N. flavoviridis, N. nivea, N. obscura, N. roseolilacina and N. sericata, and the new combination N. sublutea for a taxon formerly placed in Hypoxylon. The monotypic genus Kretzschmariella, represented by K. culmorum, is included for its morphological resemblance and possible confusion with Nemania. With the exception of K. culmorum and N. bipapillata which were already reported from the Caribbean, all other taxa are new to this region. A dichotomous identification key and a synoptic figure plate of stromata are presented</description>
+</descriptions>
+<subjects>
+  <subject xml:lang="en">Xylariales</subject>
+  <subject xml:lang="en">Xylariaceae</subject>
+  <subject xml:lang="en">Kretzschmariella</subject>
+  <subject xml:lang="en">Nemania</subject>
+  <subject xml:lang="en">Martinique</subject>
+  <subject xml:lang="en">Guadeloupe</subject>
+</subjects>
+</resource>