Skip to content

Commit 84dc9bb

Browse files
dekobondekobon
committed
Merge branch 'master' into fixes_270
2 parents 08b96be + bb1a00f commit 84dc9bb

File tree

8 files changed

+156
-7
lines changed

8 files changed

+156
-7
lines changed

‎CHANGELOG.md

+6
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file.
33
This project aims to adhere to [Semantic Versioning](http://semver.org/).
44

55

6+
## [3.x.x] - XXXX-XX-XX
7+
### Changed
8+
- The heuristics for guessing heuristics have been adjusted to give
9+
[more consistent](https://github.com/joyent/java-manta/issues/276)
10+
results across platforms.
11+
612
## [3.1.3] - 2017-06-29
713
### Fixed
814
- [`Digest` not reset in DigestedEntity](https://github.com/joyent/java-manta/pull/280)

‎java-manta-client/src/main/java/com/joyent/manta/client/multipart/AbstractMultipartManager.java

+2
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ public void validateThatThereAreSequentialPartNumbers(final UPLOAD upload)
6161
MantaMultipartException e = new MantaMultipartException(
6262
"Missing part of multipart upload");
6363
e.setContextValue("missing_part", memo);
64+
e.setContextValue("uploadId", upload.getId());
65+
e.setContextValue("path", upload.getPath());
6466
throw e;
6567
}
6668

‎java-manta-client/src/main/java/com/joyent/manta/client/multipart/EncryptionState.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,9 @@ ByteArrayOutputStream remainderAndLastPartAuth() throws IOException {
133133
if (isLastPartAuthWritten()) {
134134
final String msg = "final CSE auth already written (complete called multiple times or "
135135
+ "parts below min size)";
136-
throw new MantaMultipartException(new IllegalStateException(msg));
136+
MantaMultipartException mme = new MantaMultipartException(new IllegalStateException(msg));
137+
mme.setContextValue("lastPartNumber", getLastPartNumber());
138+
throw mme;
137139
}
138140
ByteArrayOutputStream remainderStream = new ByteArrayOutputStream();
139141
getMultipartStream().setNext(remainderStream);

‎java-manta-client/src/main/java/com/joyent/manta/client/multipart/ServerSideMultipartManager.java

+8
Original file line numberDiff line numberDiff line change
@@ -618,6 +618,14 @@ public void complete(final ServerSideMultipartUpload upload,
618618
LOGGER.info("Multipart upload [{}] for file [{}] from [{}] parts has completed",
619619
upload.getId(), location.getValue(), numParts);
620620
}
621+
} catch (MantaMultipartException | MantaClientHttpResponseException me) {
622+
// Already annotated
623+
throw me;
624+
} catch (Exception e) {
625+
String msg = "Error initiating multipart upload completion";
626+
MantaMultipartException me = new MantaMultipartException(msg, e);
627+
annotateException(me, post, null, path, jsonRequest);
628+
throw me;
621629
}
622630
}
623631

‎java-manta-client/src/main/java/com/joyent/manta/http/ContentTypeLookup.java

+16-5
Original file line numberDiff line numberDiff line change
@@ -74,12 +74,23 @@ public static ContentType findOrDefaultContentType(final MantaHttpHeaders header
7474
String type = ObjectUtils.firstNonNull(
7575
// Use explicitly set headers if available
7676
headerContentType,
77-
// Probe using the JVM default detection method
78-
Files.probeContentType(file.toPath()),
79-
// Detect based on destination filename
77+
// Detect based on destination and then source
78+
// filename. URLConnection uses a property list
79+
// bundled with the JVM and is expected to be
80+
// consistent on all platforms. As implied by the
81+
// method name, the contents of the file are not
82+
// considered.
8083
URLConnection.guessContentTypeFromName(filename),
81-
// Detect based on source filename
82-
URLConnection.guessContentTypeFromName(file.getName())
84+
URLConnection.guessContentTypeFromName(file.getName()),
85+
// Probe using the JVM default detection method. The
86+
// detection methods vary across platforms and may
87+
// rely on /etc/mime.types or include native libraries
88+
// such as libgio or libreal. The contents of the
89+
// file may be inspected. This check is ordered last
90+
// both for cross-platform consistency. See
91+
// https://github.com/joyent/java-manta/issues/276 for
92+
// further context.
93+
Files.probeContentType(file.toPath())
8394
);
8495

8596
if (type == null) {

‎java-manta-client/src/test/java/com/joyent/manta/http/ContentTypeLookupTest.java

+110
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
/*
2+
* Copyright (c) 2017, Joyent, Inc. All rights reserved.
3+
*
4+
* This Source Code Form is subject to the terms of the Mozilla Public
5+
* License, v. 2.0. If a copy of the MPL was not distributed with this
6+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
7+
*/
8+
package com.joyent.manta.http;
9+
10+
import org.apache.commons.io.FileUtils;
11+
import org.apache.http.Header;
12+
import org.apache.http.HttpHeaders;
13+
import org.apache.http.entity.ContentType;
14+
import org.apache.http.entity.ContentType;
15+
import org.apache.http.message.BasicHeader;
16+
import org.testng.Assert;
17+
import org.testng.annotations.Test;
18+
19+
import java.io.File;
20+
21+
@Test
22+
public class ContentTypeLookupTest {
23+
private static final Header[] EXAMPLE_HEADERS = new Header[] {
24+
new BasicHeader("Last-Modified","Fri, 09 Dec 2016 22:09:44 GMT"),
25+
new BasicHeader("Result-Set-Size", "0"),
26+
new BasicHeader("Date", "Fri, 09 Dec 2016 22:09:44 GMT"),
27+
new BasicHeader("Server", "Manta"),
28+
new BasicHeader("x-request-id", "17d243b4-f9a7-4042-bc21-7322fb40fc1c"),
29+
new BasicHeader("x-response-time", "16"),
30+
new BasicHeader("x-server-name", "02d02889-cd80-4ac1-bc0c-4775b86661e4"),
31+
new BasicHeader("Connection", "keep-alive")};
32+
33+
public void canFindDefault() {
34+
MantaHttpHeaders headers = new MantaHttpHeaders(EXAMPLE_HEADERS);
35+
ContentType troff = ContentType.create("application/x-troff");
36+
ContentType jsonStream = ContentType.create("application/x-json-stream");
37+
38+
Assert.assertEquals(ContentTypeLookup.findOrDefaultContentType(null, troff).getMimeType(),
39+
troff.getMimeType());
40+
Assert.assertEquals(ContentTypeLookup.findOrDefaultContentType(headers, troff).getMimeType(),
41+
troff.getMimeType());
42+
headers.put("Content-Type", "application/x-json-stream; type=directory");
43+
Assert.assertEquals(ContentTypeLookup.findOrDefaultContentType(headers, troff).getMimeType(),
44+
jsonStream.getMimeType());
45+
}
46+
47+
// These test will fail on some platforms where libgio will return
48+
// "text/plain" for empty files instead of indicating an unknown type.
49+
@Test(enabled = false)
50+
public void canfindByMultipleMethodsNull() throws Exception {
51+
MantaHttpHeaders headers = new MantaHttpHeaders(EXAMPLE_HEADERS);
52+
ContentType troff = ContentType.create("application/x-troff");
53+
ContentType jsonStream = ContentType.create("application/x-json-stream");
54+
55+
File temp = File.createTempFile("upload", ".unknown");
56+
FileUtils.forceDeleteOnExit(temp);
57+
Assert.assertNull(ContentTypeLookup.findOrDefaultContentType(null, "/stor/unknown", temp, null));
58+
59+
temp = File.createTempFile("upload", ".unknown");
60+
FileUtils.forceDeleteOnExit(temp);
61+
Assert.assertNull(ContentTypeLookup.findOrDefaultContentType(headers, "/stor/unknown", temp, null));
62+
}
63+
64+
public void canfindByMultipleMethods() throws Exception {
65+
MantaHttpHeaders headers = new MantaHttpHeaders(EXAMPLE_HEADERS);
66+
ContentType troff = ContentType.create("application/x-troff");
67+
ContentType jsonStream = ContentType.create("application/x-json-stream");
68+
69+
File temp = File.createTempFile("upload", ".jpeg");
70+
FileUtils.forceDeleteOnExit(temp);
71+
Assert.assertEquals(ContentTypeLookup.findOrDefaultContentType(headers,
72+
"/stor/unknown",
73+
temp,
74+
troff).getMimeType(),
75+
"image/jpeg");
76+
headers.put("Content-Type", "application/x-json-stream; type=directory");
77+
temp = File.createTempFile("upload", ".jpeg");
78+
FileUtils.forceDeleteOnExit(temp);
79+
Assert.assertEquals(ContentTypeLookup.findOrDefaultContentType(headers,
80+
"/stor/unknown",
81+
temp,
82+
troff).getMimeType(),
83+
jsonStream.getMimeType());
84+
}
85+
86+
public void canFindByFilename() {
87+
MantaHttpHeaders headers = new MantaHttpHeaders(EXAMPLE_HEADERS);
88+
ContentType troff = ContentType.create("application/x-troff");
89+
ContentType jsonStream = ContentType.create("application/x-json-stream");
90+
91+
Assert.assertNull(ContentTypeLookup.findOrDefaultContentType(null, "/tmp/unknown", null));
92+
Assert.assertNull(ContentTypeLookup.findOrDefaultContentType(headers, "/tmp/unknown", null));
93+
94+
Assert.assertEquals(ContentTypeLookup.findOrDefaultContentType(headers,
95+
"/tmp/unknown",
96+
troff).getMimeType(),
97+
troff.getMimeType());
98+
99+
Assert.assertEquals(ContentTypeLookup.findOrDefaultContentType(headers,
100+
"/tmp/foo.jpeg",
101+
troff).getMimeType(),
102+
"image/jpeg");
103+
headers.put("Content-Type", "application/x-json-stream; type=directory");
104+
Assert.assertEquals(ContentTypeLookup.findOrDefaultContentType(headers,
105+
"/tmp/foo.jpeg",
106+
troff).getMimeType(),
107+
jsonStream.getMimeType());
108+
109+
}
110+
}

‎java-manta-it/src/test/java/com/joyent/manta/client/multipart/EncryptedServerSideMultipartManagerIT.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -444,10 +444,10 @@ public void cantUploadSmallMultipartString() throws IOException {
444444
assert exception.getCause() instanceof IllegalStateException;
445445
}
446446
Assert.assertEquals(uploadedParts.size(), 1, "only first small upload should succeed");
447-
multipart.validateThatThereAreSequentialPartNumbers(upload);
448447

449448
// This "completes" but only uploads one part due to the previous exceptions
450449
multipart.complete(upload, uploadedParts);
450+
Assert.assertEquals(mantaClient.getAsString(path), "Hello ");
451451
}
452452

453453
public final void doubleCompleteFails() throws IOException {
@@ -461,6 +461,9 @@ public final void doubleCompleteFails() throws IOException {
461461
MantaMultipartUploadTuple[] parts = new MantaMultipartUploadTuple[] { part1 };
462462
Stream<MantaMultipartUploadTuple> partsStream = Arrays.stream(parts);
463463
multipart.complete(upload, partsStream);
464+
try (MantaObjectInputStream in = mantaClient.getAsInputStream(path)) {
465+
Assert.assertEquals(IOUtils.toByteArray(in), content);
466+
}
464467
Assert.assertThrows(IllegalStateException.class,
465468
() -> {
466469
multipart.complete(upload, partsStream);

‎pom.xml

+7
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,13 @@
4848
<organization>Joyent</organization>
4949
<organizationUrl>joyent.com</organizationUrl>
5050
</developer>
51+
<developer>
52+
<name>Tomas Celaya</name>
53+
<email>tjcelaya@joyent.com</email>
54+
<organization>Joyent</organization>
55+
<organizationUrl>joyent.com</organizationUrl>
56+
<timezone>US/Pacific</timezone>
57+
</developer>
5158
</developers>
5259
<contributors>
5360
<contributor>

0 commit comments

Comments
 (0)