fixes OLINGO-227

fit/pom.xml

@@ -94,18 +94,6 @@
     <defaultGoal>clean package cargo:run</defaultGoal>
     
     <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-resources-plugin</artifactId>
-        <version>2.6</version>
-        <configuration>
-          <useDefaultDelimiters>false</useDefaultDelimiters>
-          <delimiters>
-            <delimiter>${*}</delimiter>
-          </delimiters>
-        </configuration>
-      </plugin>
-      
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-war-plugin</artifactId>

fit/src/main/java/org/apache/olingo/fit/AbstractServices.java

@@ -1017,15 +1017,17 @@ public abstract class AbstractServices {
 
     final String basePath = Commons.getEntityBasePath(entitySetName, entityId);
 
-    InputStream stream = FSManager.instance(getVersion()).readFile(
-            basePath + Constants.get(getVersion(), ConstantKey.ENTITY), acceptType == null || acceptType == Accept.TEXT
-            ? Accept.XML : acceptType);
-
     final AbstractUtilities utils = getUtilities(acceptType);
 
     final List<String> pathElements = Arrays.asList(path.split("\\/"));
 
+    InputStream stream;
+
     if (searchForValue) {
+      stream = FSManager.instance(getVersion()).readFile(
+              basePath + Constants.get(getVersion(), ConstantKey.ENTITY),
+              acceptType == null || acceptType == Accept.TEXT ? Accept.XML : acceptType);
+
       stream = utils.getPropertyValue(stream, pathElements);
     } else {
       String edmType = xml.getEdmTypeFromAtom(entitySetName, entityId, pathElements);

fit/src/main/java/org/apache/olingo/fit/utils/AbstractJSONUtilities.java

@@ -63,7 +63,7 @@ public abstract class AbstractJSONUtilities extends AbstractUtilities {
               new TextNode(Commons.getLinksURI(version, entitySetName, entitykey, link)));
     }
 
-    return IOUtils.toInputStream(srcNode.toString());
+    return IOUtils.toInputStream(srcNode.toString(), "UTf-8");
   }
 
   @Override
@@ -126,7 +126,7 @@ public abstract class AbstractJSONUtilities extends AbstractUtilities {
 
         links.addLinks(title, hrefs);
       } else if (Commons.linkInfo.get(version).exists(entitySetName, field.getKey())) {
-        links.addInlines(field.getKey(), IOUtils.toInputStream(field.getValue().toString()));
+        links.addInlines(field.getKey(), IOUtils.toInputStream(field.getValue().toString(), "UTf-8"));
       }
     }
 
@@ -171,7 +171,7 @@ public abstract class AbstractJSONUtilities extends AbstractUtilities {
             Constants.get(version, ConstantKey.JSON_EDITLINK_NAME), new TextNode(
             Constants.get(version, ConstantKey.DEFAULT_SERVICE_URL) + entitySetName + "(" + entityKey + ")"));
 
-    return IOUtils.toInputStream(srcNode.toString());
+    return IOUtils.toInputStream(srcNode.toString(), "UTf-8");
   }
 
   @Override
@@ -303,7 +303,7 @@ public abstract class AbstractJSONUtilities extends AbstractUtilities {
 
     srcNode.retain(retain);
 
-    return IOUtils.toInputStream(srcNode.toString());
+    return IOUtils.toInputStream(srcNode.toString(), "UTf-8");
   }
 
   @Override
@@ -352,7 +352,7 @@ public abstract class AbstractJSONUtilities extends AbstractUtilities {
       node.set(Constants.get(version, ConstantKey.JSON_NEXTLINK_NAME), new TextNode(next));
     }
 
-    return IOUtils.toInputStream(node.toString());
+    return IOUtils.toInputStream(node.toString(), "UTf-8");
   }
 
   @Override
@@ -375,7 +375,7 @@ public abstract class AbstractJSONUtilities extends AbstractUtilities {
       toBeChangedNode.set(linkName + Constants.get(version, ConstantKey.JSON_NEXTLINK_SUFFIX), next);
     }
 
-    return IOUtils.toInputStream(toBeChangedNode.toString());
+    return IOUtils.toInputStream(toBeChangedNode.toString(), "UTf-8");
   }
 
   @Override
@@ -388,7 +388,7 @@ public abstract class AbstractJSONUtilities extends AbstractUtilities {
     final Iterator<Map.Entry<String, JsonNode>> fields = srcObject.fields();
     while (fields.hasNext()) {
       final Map.Entry<String, JsonNode> field = fields.next();
-      res.put(field.getKey(), IOUtils.toInputStream(field.getValue().toString()));
+      res.put(field.getKey(), IOUtils.toInputStream(field.getValue().toString(), "UTf-8"));
     }
 
     return res;
@@ -406,7 +406,7 @@ public abstract class AbstractJSONUtilities extends AbstractUtilities {
       toBeChangedObject.set(property.getKey(), propertyNode);
     }
 
-    return IOUtils.toInputStream(toBeChangedObject.toString());
+    return IOUtils.toInputStream(toBeChangedObject.toString(), "UTf-8");
   }
 
   @Override
@@ -452,7 +452,7 @@ public abstract class AbstractJSONUtilities extends AbstractUtilities {
     IOUtils.closeQuietly(content);
 
     srcNode.set(Constants.get(version, ConstantKey.JSON_EDITLINK_NAME), new TextNode(href));
-    return IOUtils.toInputStream(srcNode.toString());
+    return IOUtils.toInputStream(srcNode.toString(), "UTf-8");
   }
 
   @Override
@@ -481,7 +481,7 @@ public abstract class AbstractJSONUtilities extends AbstractUtilities {
 
     ((ObjectNode) node).set(path.get(path.size() - 1), replacementNode);
 
-    return IOUtils.toInputStream(srcNode.toString());
+    return IOUtils.toInputStream(srcNode.toString(), "UTf-8");
   }
 
   @Override
@@ -500,6 +500,6 @@ public abstract class AbstractJSONUtilities extends AbstractUtilities {
 
     ((ObjectNode) node).set(path.get(path.size() - 1), null);
 
-    return IOUtils.toInputStream(srcNode.toString());
+    return IOUtils.toInputStream(srcNode.toString(), "UTf-8");
   }
 }

fit/src/main/java/org/apache/olingo/fit/utils/AbstractUtilities.java

@@ -387,18 +387,14 @@ public abstract class AbstractUtilities {
       builder.header("ETag", etag);
     }
 
-    if (accept != null) {
-      builder.header("Content-Type", accept.toString(version));
-    } else {
-      builder.header("Content-Type", "*/*");
-    }
-
     if (status != null) {
       builder.status(status);
     }
 
     int contentLength = 0;
 
+    String contentTypeEncoding = StringUtils.EMPTY;
+
     if (entity != null) {
       try {
         final InputStream toBeStreamedBack;
@@ -415,12 +411,15 @@ public abstract class AbstractUtilities {
 
         contentLength = bos.size();
         builder.entity(new ByteArrayInputStream(bos.toByteArray()));
+
+        contentTypeEncoding = ";odata.streaming=true;charset=utf-8";
       } catch (IOException ioe) {
         LOG.error("Error streaming response entity back", ioe);
       }
     }
 
     builder.header("Content-Length", contentLength);
+    builder.header("Content-Type", (accept == null ? "*/*" : accept.toString(version)) + contentTypeEncoding);
 
     return builder.build();
   }

fit/src/main/java/org/apache/olingo/fit/utils/AbstractXMLUtilities.java

@@ -24,7 +24,7 @@ import java.io.File;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
-import java.io.StringWriter;
+import java.nio.charset.Charset;
 import java.util.AbstractMap;
 import java.util.AbstractMap.SimpleEntry;
 import java.util.ArrayList;
@@ -75,7 +75,7 @@ public abstract class AbstractXMLUtilities extends AbstractUtilities {
       ifactory = XMLInputFactory.newInstance();
     }
     ifactory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, false);
-    return ifactory.createXMLEventReader(is);
+    return ifactory.createXMLEventReader(is, "utf-8");
   }
 
   protected static XMLEventWriter getEventWriter(final OutputStream os) throws XMLStreamException {
@@ -83,7 +83,7 @@ public abstract class AbstractXMLUtilities extends AbstractUtilities {
       ofactory = XMLOutputFactory.newInstance();
     }
 
-    return ofactory.createXMLEventWriter(os);
+    return ofactory.createXMLEventWriter(os, "utf-8");
   }
 
   private void writeEvent(final XMLEvent event, final XMLEventWriter writer) {
@@ -369,7 +369,9 @@ public abstract class AbstractXMLUtilities extends AbstractUtilities {
     final XmlElement res = new XmlElement();
     res.setStart(start);
 
-    StringWriter content = new StringWriter();
+    final Charset encoding = Charset.forName("UTF-8");
+    final ByteArrayOutputStream content = new ByteArrayOutputStream();
+    final OutputStreamWriter writer = new OutputStreamWriter(content, encoding);
 
     int depth = 1;
 
@@ -385,14 +387,14 @@ public abstract class AbstractXMLUtilities extends AbstractUtilities {
       if (depth == 0) {
         res.setEnd(event.asEndElement());
       } else {
-        event.writeAsEncodedUnicode(content);
+        event.writeAsEncodedUnicode(writer);
       }
     }
 
-    content.flush();
+    writer.flush();
-    content.close();
+    writer.close();
 
-    res.setContent(new ByteArrayInputStream(content.toString().getBytes()));
+    res.setContent(new ByteArrayInputStream(content.toByteArray()));
 
     return res;
   }
@@ -851,24 +853,28 @@ public abstract class AbstractXMLUtilities extends AbstractUtilities {
       throw new NotFoundException();
     }
 
+    final Charset encoding = Charset.forName("UTF-8");
+
     final ByteArrayOutputStream bos = new ByteArrayOutputStream();
+    final OutputStreamWriter writer = new OutputStreamWriter(bos, encoding);
+
+    writer.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>".toCharArray());
 
     if (forceFeed || links.size() > 1) {
       // build a feed
-      bos.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>".getBytes());
 
-      bos.write(("<feed xml:base=\"" + Constants.get(version, ConstantKey.DEFAULT_SERVICE_URL) + "\" "
+      writer.write(("<feed xml:base=\"" + Constants.get(version, ConstantKey.DEFAULT_SERVICE_URL) + "\" "
               + "xmlns=\"http://www.w3.org/2005/Atom\" "
               + "xmlns:d=\"http://schemas.microsoft.com/ado/2007/08/dataservices\" "
               + "xmlns:m=\"http://schemas.microsoft.com/ado/2007/08/dataservices/metadata\">")
-              .getBytes());
+              .toCharArray());
 
-      bos.write(("<id>" + Constants.get(version, ConstantKey.DEFAULT_SERVICE_URL) + "entityset(entityid)/" + linkName
+      writer.write(("<id>" + Constants.get(version, ConstantKey.DEFAULT_SERVICE_URL) + "entityset(entityid)/" + linkName
-              + "</id>").getBytes());
+              + "</id>").toCharArray());
 
-      bos.write(("<title type=\"text\">" + linkName + "</title>").getBytes());
+      writer.write(("<title type=\"text\">" + linkName + "</title>").toCharArray());
-      bos.write("<updated>2014-03-03T13:40:49Z</updated>".getBytes());
+      writer.write("<updated>2014-03-03T13:40:49Z</updated>".toCharArray());
-      bos.write(("<link rel=\"self\" title=\"" + linkName + "\" href=\"" + linkName + "\" />").getBytes());
+      writer.write(("<link rel=\"self\" title=\"" + linkName + "\" href=\"" + linkName + "\" />").toCharArray());
     }
 
     for (String link : links) {
@@ -882,7 +888,7 @@ public abstract class AbstractXMLUtilities extends AbstractUtilities {
                 Collections.<String>singletonList("entry"),
                 0, 1, 1).getValue();
 
-        IOUtils.copy(entry.toStream(), bos);
+        IOUtils.copy(entry.toStream(), writer, encoding);
       } catch (Exception e) {
         // log and ignore link
         LOG.warn("Error parsing uri {}", link, e);
@@ -892,12 +898,15 @@ public abstract class AbstractXMLUtilities extends AbstractUtilities {
     if (forceFeed || links.size() > 1) {
 
       if (StringUtils.isNotBlank(next)) {
-        bos.write(String.format("<link rel=\"next\" href=\"%s\" />", next).getBytes());
+        writer.write(String.format("<link rel=\"next\" href=\"%s\" />", next).toCharArray());
       }
 
-      bos.write("</feed>".getBytes());
+      writer.write("</feed>".toCharArray());
     }
 
+    writer.flush();
+    writer.close();
+
     return new ByteArrayInputStream(bos.toByteArray());
   }
 
@@ -1227,7 +1236,7 @@ public abstract class AbstractXMLUtilities extends AbstractUtilities {
     final XMLEventWriter writer = getEventWriter(bos);
 
     final XMLEventFactory eventFactory = XMLEventFactory.newInstance();
-    writer.add(eventFactory.createStartDocument("UTF-8", "1.0"));
+    writer.add(eventFactory.createStartDocument("utf-8", "1.0"));
     writer.add(property.getStart());
 
     if (property.getStart().getAttributeByName(new QName(

fit/src/main/java/org/apache/olingo/fit/utils/Commons.java

@@ -183,7 +183,7 @@ public abstract class Commons {
       links.set("value", uris);
     }
 
-    return IOUtils.toInputStream(links.toString());
+    return IOUtils.toInputStream(links.toString(), "UTf-8");
   }
 
   public static InputStream changeFormat(final InputStream is, final Accept target) {
@@ -197,7 +197,7 @@ public abstract class Commons {
       final JsonNode node =
               changeFormat((ObjectNode) mapper.readTree(new ByteArrayInputStream(bos.toByteArray())), target);
 
-      return IOUtils.toInputStream(node.toString());
+      return IOUtils.toInputStream(node.toString(), "UTF-8");
     } catch (Exception e) {
       LOG.error("Error changing format", e);
       return new ByteArrayInputStream(bos.toByteArray());

fit/src/main/java/org/apache/olingo/fit/utils/XMLEventReaderWrapper.java

@@ -20,6 +20,10 @@ package org.apache.olingo.fit.utils;
 
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
 import javax.xml.stream.XMLEventReader;
 import javax.xml.stream.XMLInputFactory;
 import javax.xml.stream.XMLStreamException;
@@ -28,6 +32,8 @@ import org.apache.commons.io.IOUtils;
 
 public class XMLEventReaderWrapper implements XMLEventReader {
 
+  private static Charset encoding = Charset.forName("UTF-8");
+
   public final static String CONTENT = "CONTENT_TAG";
 
   public final static String CONTENT_STAG = "<" + CONTENT + ">";
@@ -43,12 +49,17 @@ public class XMLEventReaderWrapper implements XMLEventReader {
     factory.setProperty(XMLInputFactory.IS_VALIDATING, false);
     factory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, false);
 
+    final CharsetDecoder decoder = encoding.newDecoder();
+    decoder.onMalformedInput(CodingErrorAction.IGNORE);
+    decoder.onUnmappableCharacter(CodingErrorAction.IGNORE);
 
-    this.wrapped = factory.createXMLEventReader(
+    InputStreamReader reader = new InputStreamReader(
-            new ByteArrayInputStream(
+            new ByteArrayInputStream((XMLEventReaderWrapper.CONTENT_STAG
-            (XMLEventReaderWrapper.CONTENT_STAG
+            + IOUtils.toString(stream, encoding).replaceAll("^<\\?xml.*\\?>", "")
-            + IOUtils.toString(stream).replaceAll("^<\\?xml.*\\?>", "")
+            + XMLEventReaderWrapper.CONTENT_ETAG).getBytes(encoding)),
-            + XMLEventReaderWrapper.CONTENT_ETAG).getBytes()));
+            decoder);
+
+    this.wrapped = factory.createXMLEventReader(reader);
 
     init();
   }

fit/src/main/java/org/apache/olingo/fit/utils/XmlElement.java

@@ -22,7 +22,9 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
 import javax.xml.stream.XMLEventReader;
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.events.EndElement;
@@ -38,6 +40,8 @@ public class XmlElement {
    */
   protected static final Logger LOG = LoggerFactory.getLogger(XmlElement.class);
 
+  private static Charset encoding = Charset.forName("UTF-8");
+
   private StartElement start;
 
   private EndElement end;
@@ -70,20 +74,26 @@ public class XmlElement {
 
   public void setContent(final InputStream content) throws IOException {
     this.content.reset();
-    IOUtils.copyLarge(content, this.content);
+
-    content.close();
+    final InputStreamReader reader = new InputStreamReader(content, encoding);
+    final OutputStreamWriter writer = new OutputStreamWriter(this.content, encoding);
+    IOUtils.copyLarge(reader, writer);
+
+    writer.flush();
+    IOUtils.closeQuietly(reader);
+    IOUtils.closeQuietly(writer);
+    IOUtils.closeQuietly(content);
   }
 
   public InputStream toStream() throws Exception {
     InputStream res;
     try {
       final ByteArrayOutputStream bos = new ByteArrayOutputStream();
-      final OutputStreamWriter osw = new OutputStreamWriter(bos);
+      final OutputStreamWriter osw = new OutputStreamWriter(bos, encoding);
 
       getStart().writeAsEncodedUnicode(osw);
-      osw.flush();
 
-      IOUtils.copy(getContent(), bos);
+      IOUtils.copy(getContent(), osw, encoding);
 
       getEnd().writeAsEncodedUnicode(osw);
       osw.flush();

lib/client-core/src/test/java/org/apache/olingo/client/core/it/v3/AsyncTestITCase.java

@@ -39,7 +39,6 @@ import org.apache.olingo.client.api.uri.CommonURIBuilder;
 import org.apache.olingo.commons.api.domain.CommonODataEntity;
 import org.apache.olingo.commons.api.domain.v3.ODataEntity;
 import org.apache.olingo.commons.api.domain.v3.ODataEntitySet;
-import org.junit.Ignore;
 import org.junit.Test;
 
 public class AsyncTestITCase extends AbstractTestITCase {

pom.xml

@@ -332,6 +332,19 @@
           </executions>
         </plugin>
         
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-resources-plugin</artifactId>
+          <version>2.6</version>
+          <configuration>
+            <encoding>UTF-8</encoding>
+            <useDefaultDelimiters>false</useDefaultDelimiters>
+            <delimiters>
+              <delimiter>${*}</delimiter>
+            </delimiters>
+          </configuration>
+        </plugin>
+
       </plugins>
     </pluginManagement>