HADOOP-18660. Filesystem Spelling Mistake (#5475). Contributed by Sebastian Baunsgaard.
Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>
This commit is contained in:
parent
2f66f0b83a
commit
6aac6cb212
|
@ -3602,9 +3602,9 @@ public abstract class FileSystem extends Configured
|
||||||
} catch (IOException | RuntimeException e) {
|
} catch (IOException | RuntimeException e) {
|
||||||
// exception raised during initialization.
|
// exception raised during initialization.
|
||||||
// log summary at warn and full stack at debug
|
// log summary at warn and full stack at debug
|
||||||
LOGGER.warn("Failed to initialize fileystem {}: {}",
|
LOGGER.warn("Failed to initialize filesystem {}: {}",
|
||||||
uri, e.toString());
|
uri, e.toString());
|
||||||
LOGGER.debug("Failed to initialize fileystem", e);
|
LOGGER.debug("Failed to initialize filesystem", e);
|
||||||
// then (robustly) close the FS, so as to invoke any
|
// then (robustly) close the FS, so as to invoke any
|
||||||
// cleanup code.
|
// cleanup code.
|
||||||
IOUtils.cleanupWithLogger(LOGGER, fs);
|
IOUtils.cleanupWithLogger(LOGGER, fs);
|
||||||
|
|
|
@ -31,7 +31,7 @@ There are a number of goals here:
|
||||||
having to invoke them.
|
having to invoke them.
|
||||||
1. Allow filesystems with their own optional per-instance features to declare
|
1. Allow filesystems with their own optional per-instance features to declare
|
||||||
whether or not they are active for the specific instance.
|
whether or not they are active for the specific instance.
|
||||||
1. Allow for fileystem connectors which work with object stores to expose the
|
1. Allow for filesystem connectors which work with object stores to expose the
|
||||||
fundamental difference in semantics of these stores (e.g: files not visible
|
fundamental difference in semantics of these stores (e.g: files not visible
|
||||||
until closed, file rename being `O(data)`), directory rename being non-atomic,
|
until closed, file rename being `O(data)`), directory rename being non-atomic,
|
||||||
etc.
|
etc.
|
||||||
|
@ -122,7 +122,7 @@ will be permitted on that path by the caller.
|
||||||
*Duration of availability*
|
*Duration of availability*
|
||||||
|
|
||||||
As the state of a remote store changes,so may path capabilities. This
|
As the state of a remote store changes,so may path capabilities. This
|
||||||
may be due to changes in the local state of the fileystem (e.g. symbolic links
|
may be due to changes in the local state of the filesystem (e.g. symbolic links
|
||||||
or mount points changing), or changes in its functionality (e.g. a feature
|
or mount points changing), or changes in its functionality (e.g. a feature
|
||||||
becoming availaible/unavailable due to operational changes, system upgrades, etc.)
|
becoming availaible/unavailable due to operational changes, system upgrades, etc.)
|
||||||
|
|
||||||
|
|
|
@ -150,7 +150,7 @@ public class HttpFSServer {
|
||||||
* @return FileSystemExecutor response
|
* @return FileSystemExecutor response
|
||||||
*
|
*
|
||||||
* @throws IOException thrown if an IO error occurs.
|
* @throws IOException thrown if an IO error occurs.
|
||||||
* @throws FileSystemAccessException thrown if a FileSystemAccess releated error occurred. Thrown
|
* @throws FileSystemAccessException thrown if a FileSystemAccess related error occurred. Thrown
|
||||||
* exceptions are handled by {@link HttpFSExceptionProvider}.
|
* exceptions are handled by {@link HttpFSExceptionProvider}.
|
||||||
*/
|
*/
|
||||||
private <T> T fsExecute(UserGroupInformation ugi, FileSystemAccess.FileSystemExecutor<T> executor)
|
private <T> T fsExecute(UserGroupInformation ugi, FileSystemAccess.FileSystemExecutor<T> executor)
|
||||||
|
@ -161,8 +161,8 @@ public class HttpFSServer {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns a filesystem instance. The fileystem instance is wired for release at the completion of
|
* Returns a filesystem instance. The filesystem instance is wired for release at the completion
|
||||||
* the current Servlet request via the {@link FileSystemReleaseFilter}.
|
* of the current Servlet request via the {@link FileSystemReleaseFilter}.
|
||||||
* <p>
|
* <p>
|
||||||
* If a do-as user is specified, the current user must be a valid proxyuser, otherwise an
|
* If a do-as user is specified, the current user must be a valid proxyuser, otherwise an
|
||||||
* <code>AccessControlException</code> will be thrown.
|
* <code>AccessControlException</code> will be thrown.
|
||||||
|
@ -173,7 +173,7 @@ public class HttpFSServer {
|
||||||
*
|
*
|
||||||
* @throws IOException thrown if an IO error occurred. Thrown exceptions are
|
* @throws IOException thrown if an IO error occurred. Thrown exceptions are
|
||||||
* handled by {@link HttpFSExceptionProvider}.
|
* handled by {@link HttpFSExceptionProvider}.
|
||||||
* @throws FileSystemAccessException thrown if a FileSystemAccess releated error occurred. Thrown
|
* @throws FileSystemAccessException thrown if a FileSystemAccess related error occurred. Thrown
|
||||||
* exceptions are handled by {@link HttpFSExceptionProvider}.
|
* exceptions are handled by {@link HttpFSExceptionProvider}.
|
||||||
*/
|
*/
|
||||||
private FileSystem createFileSystem(UserGroupInformation ugi)
|
private FileSystem createFileSystem(UserGroupInformation ugi)
|
||||||
|
|
|
@ -94,14 +94,14 @@ public abstract class FileSystemReleaseFilter implements Filter {
|
||||||
* Static method that sets the <code>FileSystem</code> to release back to
|
* Static method that sets the <code>FileSystem</code> to release back to
|
||||||
* the {@link FileSystemAccess} service on servlet request completion.
|
* the {@link FileSystemAccess} service on servlet request completion.
|
||||||
*
|
*
|
||||||
* @param fs fileystem instance.
|
* @param fs a filesystem instance.
|
||||||
*/
|
*/
|
||||||
public static void setFileSystem(FileSystem fs) {
|
public static void setFileSystem(FileSystem fs) {
|
||||||
FILE_SYSTEM_TL.set(fs);
|
FILE_SYSTEM_TL.set(fs);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Abstract method to be implemetned by concrete implementations of the
|
* Abstract method to be implemented by concrete implementations of the
|
||||||
* filter that return the {@link FileSystemAccess} service to which the filesystem
|
* filter that return the {@link FileSystemAccess} service to which the filesystem
|
||||||
* will be returned to.
|
* will be returned to.
|
||||||
*
|
*
|
||||||
|
|
|
@ -371,11 +371,11 @@ any reports saved to a report directory.
|
||||||
## <a name="summaries"></a> Collecting Job Summaries `mapreduce.manifest.committer.summary.report.directory`
|
## <a name="summaries"></a> Collecting Job Summaries `mapreduce.manifest.committer.summary.report.directory`
|
||||||
|
|
||||||
The committer can be configured to save the `_SUCCESS` summary files to a report directory,
|
The committer can be configured to save the `_SUCCESS` summary files to a report directory,
|
||||||
irrespective of whether the job succeed or failed, by setting a fileystem path in
|
irrespective of whether the job succeed or failed, by setting a filesystem path in
|
||||||
the option `mapreduce.manifest.committer.summary.report.directory`.
|
the option `mapreduce.manifest.committer.summary.report.directory`.
|
||||||
|
|
||||||
The path does not have to be on the same
|
The path does not have to be on the same
|
||||||
store/filesystem as the destination of work. For example, a local fileystem could be used.
|
store/filesystem as the destination of work. For example, a local filesystem could be used.
|
||||||
|
|
||||||
XML
|
XML
|
||||||
|
|
||||||
|
|
|
@ -204,7 +204,7 @@ public class FrameworkUploader implements Runnable {
|
||||||
} else {
|
} else {
|
||||||
LOG.warn("Cannot set replication to " +
|
LOG.warn("Cannot set replication to " +
|
||||||
initialReplication + " for path: " + targetPath +
|
initialReplication + " for path: " + targetPath +
|
||||||
" on a non-distributed fileystem " +
|
" on a non-distributed filesystem " +
|
||||||
fileSystem.getClass().getName());
|
fileSystem.getClass().getName());
|
||||||
}
|
}
|
||||||
if (targetStream == null) {
|
if (targetStream == null) {
|
||||||
|
@ -319,7 +319,7 @@ public class FrameworkUploader implements Runnable {
|
||||||
} else {
|
} else {
|
||||||
LOG.info("Cannot set replication to " +
|
LOG.info("Cannot set replication to " +
|
||||||
finalReplication + " for path: " + targetPath +
|
finalReplication + " for path: " + targetPath +
|
||||||
" on a non-distributed fileystem " +
|
" on a non-distributed filesystem " +
|
||||||
fileSystem.getClass().getName());
|
fileSystem.getClass().getName());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -88,7 +88,7 @@ public class S3AReadOpContext extends S3AOpContext {
|
||||||
* Instantiate.
|
* Instantiate.
|
||||||
* @param path path of read
|
* @param path path of read
|
||||||
* @param invoker invoker for normal retries.
|
* @param invoker invoker for normal retries.
|
||||||
* @param stats Fileystem statistics (may be null)
|
* @param stats Filesystem statistics (may be null)
|
||||||
* @param instrumentation statistics context
|
* @param instrumentation statistics context
|
||||||
* @param dstFileStatus target file status
|
* @param dstFileStatus target file status
|
||||||
* @param vectoredIOContext context for vectored read operation.
|
* @param vectoredIOContext context for vectored read operation.
|
||||||
|
|
|
@ -811,7 +811,7 @@ public abstract class S3GuardTool extends Configured implements Tool,
|
||||||
try {
|
try {
|
||||||
uri = new URI(s3Path);
|
uri = new URI(s3Path);
|
||||||
} catch (URISyntaxException e) {
|
} catch (URISyntaxException e) {
|
||||||
throw invalidArgs("Not a valid fileystem path: %s", s3Path);
|
throw invalidArgs("Not a valid filesystem path: %s", s3Path);
|
||||||
}
|
}
|
||||||
return uri;
|
return uri;
|
||||||
}
|
}
|
||||||
|
|
|
@ -832,7 +832,7 @@ This tests marshalling and unmarshalling of tokens identifiers.
|
||||||
|
|
||||||
Tests the lifecycle of session tokens.
|
Tests the lifecycle of session tokens.
|
||||||
|
|
||||||
#### Integration Test `ITestSessionDelegationInFileystem`.
|
#### Integration Test `ITestSessionDelegationInFilesystem`
|
||||||
|
|
||||||
This collects DTs from one filesystem, and uses that to create a new FS instance and
|
This collects DTs from one filesystem, and uses that to create a new FS instance and
|
||||||
then perform filesystem operations. A miniKDC is instantiated.
|
then perform filesystem operations. A miniKDC is instantiated.
|
||||||
|
@ -841,7 +841,7 @@ then perform filesystem operations. A miniKDC is instantiated.
|
||||||
the second instance is picking up the DT information.
|
the second instance is picking up the DT information.
|
||||||
* `UserGroupInformation.reset()` can be used to reset user secrets after every test
|
* `UserGroupInformation.reset()` can be used to reset user secrets after every test
|
||||||
case (e.g. teardown), so that issued DTs from one test case do not contaminate the next.
|
case (e.g. teardown), so that issued DTs from one test case do not contaminate the next.
|
||||||
* It's subclass, `ITestRoleDelegationInFileystem` adds a check that the current credentials
|
* It's subclass, `ITestRoleDelegationInFilesystem` adds a check that the current credentials
|
||||||
in the DT cannot be used to access data on other buckets —that is, the active
|
in the DT cannot be used to access data on other buckets —that is, the active
|
||||||
session really is restricted to the target bucket.
|
session really is restricted to the target bucket.
|
||||||
|
|
||||||
|
|
|
@ -677,7 +677,7 @@ warning that the SDK resolution chain is in use:
|
||||||
S3A filesystem client is using the SDK region resolution chain.
|
S3A filesystem client is using the SDK region resolution chain.
|
||||||
|
|
||||||
2021-06-23 19:56:56,073 [main] WARN fs.FileSystem (FileSystem.java:createFileSystem(3464)) -
|
2021-06-23 19:56:56,073 [main] WARN fs.FileSystem (FileSystem.java:createFileSystem(3464)) -
|
||||||
Failed to initialize fileystem s3a://osm-pds/planet:
|
Failed to initialize filesystem s3a://osm-pds/planet:
|
||||||
org.apache.hadoop.fs.s3a.AWSClientIOException: creating AWS S3 client on s3a://osm-pds:
|
org.apache.hadoop.fs.s3a.AWSClientIOException: creating AWS S3 client on s3a://osm-pds:
|
||||||
com.amazonaws.SdkClientException: Unable to find a region via the region provider chain.
|
com.amazonaws.SdkClientException: Unable to find a region via the region provider chain.
|
||||||
Must provide an explicit region in the builder or setup environment to supply a region.:
|
Must provide an explicit region in the builder or setup environment to supply a region.:
|
||||||
|
@ -1224,7 +1224,7 @@ KMS key ID is required for CSE-KMS to encrypt data, not providing one leads
|
||||||
to failure.
|
to failure.
|
||||||
|
|
||||||
```
|
```
|
||||||
2021-07-07 11:33:04,550 WARN fs.FileSystem: Failed to initialize fileystem
|
2021-07-07 11:33:04,550 WARN fs.FileSystem: Failed to initialize filesystem
|
||||||
s3a://ap-south-cse/: java.lang.IllegalArgumentException: CSE-KMS
|
s3a://ap-south-cse/: java.lang.IllegalArgumentException: CSE-KMS
|
||||||
method requires KMS key ID. Use fs.s3a.encryption.key property to set it.
|
method requires KMS key ID. Use fs.s3a.encryption.key property to set it.
|
||||||
-ls: CSE-KMS method requires KMS key ID. Use fs.s3a.encryption.key property to
|
-ls: CSE-KMS method requires KMS key ID. Use fs.s3a.encryption.key property to
|
||||||
|
|
|
@ -32,8 +32,8 @@ import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
||||||
* Subclass of the session test which checks roles; only works if
|
* Subclass of the session test which checks roles; only works if
|
||||||
* a role ARN has been declared.
|
* a role ARN has been declared.
|
||||||
*/
|
*/
|
||||||
public class ITestRoleDelegationInFileystem extends
|
public class ITestRoleDelegationInFilesystem extends
|
||||||
ITestSessionDelegationInFileystem {
|
ITestSessionDelegationInFilesystem {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setup() throws Exception {
|
public void setup() throws Exception {
|
|
@ -87,10 +87,10 @@ import static org.hamcrest.collection.IsCollectionWithSize.hasSize;
|
||||||
* UGI to be initialized with security enabled.
|
* UGI to be initialized with security enabled.
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("StaticNonFinalField")
|
@SuppressWarnings("StaticNonFinalField")
|
||||||
public class ITestSessionDelegationInFileystem extends AbstractDelegationIT {
|
public class ITestSessionDelegationInFilesystem extends AbstractDelegationIT {
|
||||||
|
|
||||||
private static final Logger LOG =
|
private static final Logger LOG =
|
||||||
LoggerFactory.getLogger(ITestSessionDelegationInFileystem.class);
|
LoggerFactory.getLogger(ITestSessionDelegationInFilesystem.class);
|
||||||
|
|
||||||
private static MiniKerberizedHadoopCluster cluster;
|
private static MiniKerberizedHadoopCluster cluster;
|
||||||
|
|
||||||
|
@ -595,7 +595,7 @@ public class ITestSessionDelegationInFileystem extends AbstractDelegationIT {
|
||||||
.withEndpoint(DEFAULT_ENDPOINT)
|
.withEndpoint(DEFAULT_ENDPOINT)
|
||||||
.withMetrics(new EmptyS3AStatisticsContext()
|
.withMetrics(new EmptyS3AStatisticsContext()
|
||||||
.newStatisticsFromAwsSdk())
|
.newStatisticsFromAwsSdk())
|
||||||
.withUserAgentSuffix("ITestSessionDelegationInFileystem");
|
.withUserAgentSuffix("ITestSessionDelegationInFilesystem");
|
||||||
AmazonS3 s3 = factory.createS3Client(landsat, parameters);
|
AmazonS3 s3 = factory.createS3Client(landsat, parameters);
|
||||||
|
|
||||||
return Invoker.once("HEAD", host,
|
return Invoker.once("HEAD", host,
|
Loading…
Reference in New Issue