HDDS-680. Provide web based bucket browser.

Contributed by Elek, Marton.

(cherry picked from commit b22651e949)
This commit is contained in:
Anu Engineer 2018-10-19 08:48:11 -07:00
parent fac987b9a1
commit 63375af8c1
4 changed files with 646 additions and 5 deletions

View File

@ -613,3 +613,11 @@ which has the following notices:
Written by Doug Lea with assistance from members of JCP JSR-166
Expert Group and released to the public domain, as explained at
http://creativecommons.org/publicdomain/zero/1.0/
The source and binary distribution of this product bundles modified version of
github.com/awslabs/aws-js-s3-explorer licensed under Apache 2.0 license
with the following notice:
AWS JavaScript S3 Explorer
Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.

View File

@ -27,9 +27,11 @@ import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import java.io.IOException;
import java.io.InputStream;
import java.time.Instant;
import java.util.Iterator;
@ -60,15 +62,25 @@ public class BucketEndpoint extends EndpointBase {
* for more details.
*/
@GET
public ListObjectResponse list(
public Response list(
@PathParam("bucket") String bucketName,
@QueryParam("delimiter") String delimiter,
@QueryParam("encoding-type") String encodingType,
@QueryParam("marker") String marker,
@DefaultValue("1000") @QueryParam("max-keys") int maxKeys,
@QueryParam("prefix") String prefix,
@QueryParam("browser") String browser,
@Context HttpHeaders hh) throws OS3Exception, IOException {
if (browser != null) {
try (InputStream browserPage = getClass()
.getResourceAsStream("/browser.html")) {
return Response.ok(browserPage,
MediaType.TEXT_HTML_TYPE)
.build();
}
}
if (delimiter == null) {
delimiter = "/";
}
@ -125,7 +137,7 @@ public class BucketEndpoint extends EndpointBase {
}
response.setKeyCount(
response.getCommonPrefixes().size() + response.getContents().size());
return response;
return Response.ok(response).build();
}
@PUT

View File

@ -0,0 +1,617 @@
<!DOCTYPE html>
<!--
Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License").
You may not use this file except in compliance with the License. A copy
of the License is located at
https://aws.amazon.com/apache2.0/
or in the "license" file accompanying this file. This file is distributed
on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
either express or implied. See the License for the specific language governing
permissions and limitations under the License.
-->
<html lang="en">
<head>
<title>AWS S3 Explorer</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="shortcut icon" href="https://aws.amazon.com/favicon.ico">
<link rel="stylesheet"
href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">
<link rel="stylesheet"
href="https://use.fontawesome.com/releases/v5.2.0/css/all.css">
<link rel="stylesheet"
href="https://cdn.datatables.net/plug-ins/f2c75b7247b/integration/bootstrap/3/dataTables.bootstrap.css">
<style type="text/css">
#wrapper {
padding-left: 0;
}
#page-wrapper {
width: 100%;
padding: 5px 15px;
}
#tb-s3objects {
width: 100% !Important;
}
body {
font: 14px "Lucida Grande", Helvetica, Arial, sans-serif;
}
td {
font: 12px "Lucida Grande", Helvetica, Arial, sans-serif;
}
</style>
</head>
<!-- DEBUG: Enable this for red outline on all elements -->
<!-- <style media="screen" type="text/css"> * { outline: 1px red solid; } </style> -->
<body>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<div class="panel panel-primary">
<!-- Panel including bucket/folder information and controls -->
<div class="panel-heading clearfix">
<!-- Bucket selection and breadcrumbs -->
<div class="btn-group pull-left">
<div class="pull-left">
Ozone S3 Explorer&nbsp;
</div>
<!-- Bucket breadcrumbs -->
<div class="btn pull-right">
<ul id="breadcrumb"
class="btn breadcrumb pull-right">
<li class="active dropdown">
<a href="#">&lt;bucket&gt;</a>
</li>
</ul>
</div>
</div>
<!-- Folder/Bucket radio group and progress spinner -->
<div class="btn-group pull-right">
<div class="checkbox pull-left">
<label>
<input type="checkbox" id="hidefolders">&nbsp;Hide
folders?
</label>
<!-- Folder/Bucket radio group -->
<div class="btn-group" data-toggle="buttons">
<label class="btn btn-primary active"
title="View all objects in folder">
<i class="fa fa-angle-double-up"></i>
<input type="radio" name="optionsdepth"
value="folder" id="optionfolder"
checked>&nbsp;Folder
</label>
<label class="btn btn-primary"
title="View all objects in bucket">
<i class="fa fa-angle-double-down"></i>
<input type="radio" name="optionsdepth"
value="bucket" id="optionbucket">&nbsp;Bucket
</label>
</div>
</div>
<!-- Dual purpose: progress spinner and refresh button -->
<div class="btn-group pull-right" id="refresh">
<span id="bucket-loader" style="cursor: pointer;"
class="btn fa fa-refresh fa-2x pull-left"
title="Refresh"></span>
<span id="badgecount"
class="badge pull-right">42</span>
</div>
</div>
</div>
<!-- Panel including S3 object table -->
<div class="panel-body">
<table class="table table-bordered table-hover table-striped"
id="tb-s3objects">
<thead>
<tr>
<th>Object</th>
<th>Folder</th>
<th>Last Modified</th>
<th>Timestamp</th>
<th>Size</th>
</tr>
</thead>
<tbody id="tbody-s3objects"></tbody>
</table>
</div>
</div>
</div>
</div>
</div>
</body>
</html>
<script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/bootbox.js/4.4.0/bootbox.min.js"></script>
<script src="https://sdk.amazonaws.com/js/aws-sdk-2.207.0.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.22.0/moment.min.js"></script>
<script src="https://cdn.datatables.net/1.10.5/js/jquery.dataTables.min.js"></script>
<script src="https://cdn.datatables.net/plug-ins/f2c75b7247b/integration/bootstrap/3/dataTables.bootstrap.js"></script>
<script type="text/javascript">
var bucket;
var endpoint = document.location.protocol + '//' + document.location.host
if (document.location.pathname.length > 0) {
bucket = document.location.pathname.substring(1);
endpoint += document.location.pathname;
} else {
bucket = document.location.host.split(".")[0];
}
var s3exp_config = {
Region: '',
Bucket: bucket,
Prefix: '',
Delimiter: '/'
};
var s3exp_lister = null;
var s3exp_columns = {
key: 1,
folder: 2,
date: 3,
size: 4
};
// Initialize S3 SDK and the moment library (for time formatting utilities)
var s3 = new AWS.S3({endpoint: new AWS.Endpoint(endpoint)})
s3.config.s3BucketEndpoint = true;
moment().format();
function bytesToSize(bytes) {
var sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
if (bytes === 0) return '0 Bytes';
var ii = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)));
return Math.round(bytes / Math.pow(1024, ii), 2) + ' ' + sizes[ii];
}
// Custom startsWith function for String prototype
if (typeof String.prototype.startsWith != 'function') {
String.prototype.startsWith = function (str) {
return this.indexOf(str) == 0;
};
}
// Custom endsWith function for String prototype
if (typeof String.prototype.endsWith != 'function') {
String.prototype.endsWith = function (str) {
return this.slice(-str.length) == str;
};
}
function object2hrefvirt(bucket, key) {
var enckey = key.split('/').map(function (x) {
return encodeURIComponent(x);
}).join('/');
return endpoint + "/" + enckey;
}
function object2hrefpath(bucket, key) {
var enckey = key.split('/').map(function (x) {
return encodeURIComponent(x);
}).join('/');
return endpoint + "/" + enckey;
}
function isthisdocument(bucket, key) {
return key === "index.html";
}
function isfolder(path) {
return path.endsWith('/');
}
// Convert cars/vw/golf.png to golf.png
function fullpath2filename(path) {
return path.replace(/^.*[\\\/]/, '');
}
// Convert cars/vw/golf.png to cars/vw
function fullpath2pathname(path) {
return path.substring(0, path.lastIndexOf('/'));
}
// Convert cars/vw/ to vw/
function prefix2folder(prefix) {
var parts = prefix.split('/');
return parts[parts.length - 2] + '/';
}
// Remove hash from document URL
function removeHash() {
history.pushState("", document.title, window.location.pathname + window.location.search);
}
// We are going to generate bucket/folder breadcrumbs. The resulting HTML will
// look something like this:
//
// <li>Home</li>
// <li>Library</li>
// <li class="active">Samples</li>
//
// Note: this code is a little complex right now so it would be good to find
// a simpler way to create the breadcrumbs.
function folder2breadcrumbs(data) {
console.log('Bucket: ' + data.params.Bucket);
console.log('Prefix: ' + data.params.Prefix);
if (data.params.Prefix && data.params.Prefix.length > 0) {
console.log('Set hash: ' + data.params.Prefix);
window.location.hash = data.params.Prefix;
} else {
console.log('Remove hash');
removeHash();
}
// The parts array will contain the bucket name followed by all the
// segments of the prefix, exploded out as separate strings.
var parts = [data.params.Bucket];
if (data.params.Prefix) {
parts.push.apply(parts,
data.params.Prefix.endsWith('/') ?
data.params.Prefix.slice(0, -1).split('/') :
data.params.Prefix.split('/'));
}
console.log('Parts: ' + parts + ' (length=' + parts.length + ')');
// Empty the current breadcrumb list
$('#breadcrumb li').remove();
// Now build the new breadcrumb list
var buildprefix = '';
$.each(parts, function (ii, part) {
var ipart;
// Add the bucket (the bucket is always first)
if (ii === 0) {
var a1 = $('<a>').attr('href', '#').text(part);
ipart = $('<li>').append(a1);
a1.click(function (e) {
e.preventDefault();
console.log('Breadcrumb click bucket: ' + data.params.Bucket);
s3exp_config = {
Bucket: data.params.Bucket,
Prefix: '',
Delimiter: data.params.Delimiter
};
(s3exp_lister = s3list(s3exp_config, s3draw)).go();
});
// Else add the folders within the bucket
} else {
buildprefix += part + '/';
if (ii == parts.length - 1) {
ipart = $('<li>').addClass('active').text(part);
} else {
var a2 = $('<a>').attr('href', '#').append(part);
ipart = $('<li>').append(a2);
// Closure needed to enclose the saved S3 prefix
(function () {
var saveprefix = buildprefix;
// console.log('Part: ' + part + ' has buildprefix: ' + saveprefix);
a2.click(function (e) {
e.preventDefault();
console.log('Breadcrumb click object prefix: ' + saveprefix);
s3exp_config = {
Bucket: data.params.Bucket,
Prefix: saveprefix,
Delimiter: data.params.Delimiter
};
(s3exp_lister = s3list(s3exp_config, s3draw)).go();
});
})();
}
}
$('#breadcrumb').append(ipart);
});
}
function s3draw(data, complete) {
$('li.li-bucket').remove();
folder2breadcrumbs(data);
// Add each part of current path (S3 bucket plus folder hierarchy) into the breadcrumbs
$.each(data.CommonPrefixes, function (i, prefix) {
$('#tb-s3objects').DataTable().rows.add([{
Key: prefix.Prefix
}]);
});
// Add S3 objects to DataTable
$('#tb-s3objects').DataTable().rows.add(data.Contents).draw();
}
function s3list(config, completecb) {
console.log('s3list config: ' + JSON.stringify(config));
var params = {
Bucket: config.Bucket,
Prefix: config.Prefix,
Delimiter: config.Delimiter
};
var scope = {
Contents: [],
CommonPrefixes: [],
params: params,
stop: false,
completecb: completecb
};
return {
// This is the callback that the S3 API makes when an S3 listObjectsV2
// request completes (successfully or in error). Note that a single call
// to listObjectsV2 may not be enough to get all objects so we need to
// check if the returned data is truncated and, if so, make additional
// requests with a 'next marker' until we have all the objects.
cb: function (err, data) {
if (err) {
console.log('Error: ' + JSON.stringify(err));
console.log('Error: ' + err.stack);
scope.stop = true;
$('#bucket-loader').removeClass('fa-spin');
bootbox.alert("Error accessing S3 bucket " + scope.params.Bucket + ". Error: " + err);
} else {
// console.log('Data: ' + JSON.stringify(data));
console.log("Options: " + $("input[name='optionsdepth']:checked").val());
// Store marker before filtering data
if (data.IsTruncated) {
if (data.NextContinuationToken) {
scope.params.ContinuationToken = data.NextContinuationToken;
}
}
// Filter the folders out of the listed S3 objects
// (could probably be done more efficiently)
console.log("Filter: remove folders");
data.Contents = data.Contents.filter(function (el) {
return el.Key !== scope.params.Prefix;
});
// Accumulate the S3 objects and common prefixes
scope.Contents.push.apply(scope.Contents, data.Contents);
scope.CommonPrefixes.push.apply(scope.CommonPrefixes, data.CommonPrefixes);
// Update badge count to show number of objects read
$('#badgecount').text(scope.Contents.length + scope.CommonPrefixes.length);
if (scope.stop) {
console.log('Bucket ' + scope.params.Bucket + ' stopped');
} else if (data.IsTruncated) {
console.log('Bucket ' + scope.params.Bucket + ' truncated');
s3.makeUnauthenticatedRequest('listObjectsV2', scope.params, scope.cb);
} else {
console.log('Bucket ' + scope.params.Bucket + ' has ' + scope.Contents.length + ' objects, including ' + scope.CommonPrefixes.length + ' prefixes');
delete scope.params.ContinuationToken;
if (scope.completecb) {
scope.completecb(scope, true);
}
$('#bucket-loader').removeClass('fa-spin');
}
}
},
// Start the spinner, clear the table, make an S3 listObjectsV2 request
go: function () {
scope.cb = this.cb;
$('#bucket-loader').addClass('fa-spin');
$('#tb-s3objects').DataTable().clear();
s3.makeUnauthenticatedRequest('listObjectsV2', scope.params, this.cb);
},
stop: function () {
scope.stop = true;
delete scope.params.ContinuationToken;
if (scope.completecb) {
scope.completecb(scope, false);
}
$('#bucket-loader').removeClass('fa-spin');
}
};
}
function promptForBucketInput() {
bootbox.prompt("Please enter the S3 bucket name", function (result) {
if (result !== null) {
resetDepth();
s3exp_config = {
Bucket: result,
Delimiter: '/'
};
(s3exp_lister = s3list(s3exp_config, s3draw)).go();
}
});
}
function resetDepth() {
$('#tb-s3objects').DataTable().column(1).visible(false);
$('input[name="optionsdepth"]').val(['folder']);
$('input[name="optionsdepth"][value="bucket"]').parent().removeClass('active');
$('input[name="optionsdepth"][value="folder"]').parent().addClass('active');
}
$(document).ready(function () {
console.log('ready');
// Click handler for refresh button (to invoke manual refresh)
$('#bucket-loader').click(function (e) {
if ($('#bucket-loader').hasClass('fa-spin')) {
// To do: We need to stop the S3 list that's going on
// bootbox.alert("Stop is not yet supported.");
s3exp_lister.stop();
} else {
delete s3exp_config.ContinuationToken;
(s3exp_lister = s3list(s3exp_config, s3draw)).go();
}
});
// Click handler for bucket button (to allow user to change bucket)
$('#bucket-chooser').click(function (e) {
promptForBucketInput();
});
$('#hidefolders').click(function (e) {
$('#tb-s3objects').DataTable().draw();
});
// Folder/Bucket radio button handler
$("input:radio[name='optionsdepth']").change(function () {
console.log("Folder/Bucket option change to " + $(this).val());
console.log("Change options: " + $("input[name='optionsdepth']:checked").val());
// If user selected deep then we do need to do a full list
if ($(this).val() == 'bucket') {
console.log("Switch to bucket");
var choice = $(this).val();
$('#tb-s3objects').DataTable().column(1).visible(choice === 'bucket');
delete s3exp_config.ContinuationToken;
delete s3exp_config.Prefix;
s3exp_config.Delimiter = '';
(s3exp_lister = s3list(s3exp_config, s3draw)).go();
// Else user selected folder then can do a delimiter list
} else {
console.log("Switch to folder");
$('#tb-s3objects').DataTable().column(1).visible(false);
delete s3exp_config.ContinuationToken;
delete s3exp_config.Prefix;
s3exp_config.Delimiter = '/';
(s3exp_lister = s3list(s3exp_config, s3draw)).go();
}
});
function renderObject(data, type, full) {
if (isthisdocument(s3exp_config.Bucket, data)) {
console.log("is this document: " + data);
return fullpath2filename(data);
} else if (isfolder(data)) {
console.log("is folder: " + data);
return '<a data-s3="folder" data-prefix="' + data + '" href="' + object2hrefvirt(s3exp_config.Bucket, data) + '">' + prefix2folder(data) + '</a>';
} else {
console.log("not folder/this document: " + data);
return '<a data-s3="object" href="' + object2hrefvirt(s3exp_config.Bucket, data) + '"download="' + fullpath2filename(data) + '">' + fullpath2filename(data) + '</a>';
}
}
function renderFolder(data, type, full) {
return isfolder(data) ? "" : fullpath2pathname(data);
}
// Initial DataTable settings
$('#tb-s3objects').DataTable({
iDisplayLength: 50,
order: [
[1, 'asc'],
[0, 'asc']
],
aoColumnDefs: [{
"aTargets": [0],
"mData": "Key",
"mRender": function (data, type, full) {
return (type == 'display') ? renderObject(data, type, full) : data;
},
"sType": "key"
}, {
"aTargets": [1],
"mData": "Key",
"mRender": function (data, type, full) {
return renderFolder(data, type, full);
}
}, {
"aTargets": [2],
"mData": "LastModified",
"mRender": function (data, type, full) {
return data ? moment(data).fromNow() : "";
}
}, {
"aTargets": [3],
"mData": "LastModified",
"mRender": function (data, type, full) {
return data ? moment(data).local().format('YYYY-MM-DD HH:mm:ss') : "";
}
}, {
"aTargets": [4],
"mData": function (source, type, val) {
return source.Size ? ((type == 'display') ? bytesToSize(source.Size) : source.Size) : "";
}
},]
});
$('#tb-s3objects').DataTable().column(s3exp_columns.key).visible(false);
console.log("jQuery version=" + $.fn.jquery);
// Custom sort for the Key column so that folders appear before objects
$.fn.dataTableExt.oSort['key-asc'] = function (a, b) {
var x = (isfolder(a) ? "0-" + a : "1-" + a).toLowerCase();
var y = (isfolder(b) ? "0-" + b : "1-" + b).toLowerCase();
return ((x < y) ? -1 : ((x > y) ? 1 : 0));
};
$.fn.dataTableExt.oSort['key-desc'] = function (a, b) {
var x = (isfolder(a) ? "1-" + a : "0-" + a).toLowerCase();
var y = (isfolder(b) ? "1-" + b : "0-" + b).toLowerCase();
return ((x < y) ? 1 : ((x > y) ? -1 : 0));
};
// Allow user to hide folders
$.fn.dataTableExt.afnFiltering.push(function (oSettings, aData, iDataIndex) {
console.log("hide folders");
return $('#hidefolders').is(':checked') ? !isfolder(aData[0]) : true;
});
// Delegated event handler for S3 object/folder clicks. This is delegated
// because the object/folder rows are added dynamically and we do not want
// to have to assign click handlers to each and every row.
$('#tb-s3objects').on('click', 'a', function (event) {
event.preventDefault();
var target = event.target;
console.log("target href=" + target.href);
console.log("target dataset=" + JSON.stringify(target.dataset));
// If the user has clicked on a folder then navigate into that folder
if (target.dataset.s3 === "folder") {
resetDepth();
delete s3exp_config.ContinuationToken;
s3exp_config.Prefix = target.dataset.prefix;
s3exp_config.Delimiter = $("input[name='optionsdepth']:checked").val() == "folder" ? "/" : "";
(s3exp_lister = s3list(s3exp_config, s3draw)).go();
// Else user has clicked on an object so download it in new window/tab
} else {
window.open(target.href, '_blank');
}
return false;
});
if (window.location.hash) {
console.log("Location hash=" + window.location.hash);
s3exp_config.Prefix = window.location.hash.substring(1);
}
// Do initial bucket list
(s3exp_lister = s3list(s3exp_config, s3draw)).go();
});
</script>

View File

@ -44,7 +44,9 @@ public class TestBucketGet {
getBucket.setClient(client);
ListObjectResponse getBucketResponse =
getBucket.list("b1", "/", null, null, 100, "", null);
(ListObjectResponse) getBucket
.list("b1", "/", null, null, 100, "", null, null)
.getEntity();
Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size());
Assert.assertEquals("dir1/",
@ -66,7 +68,8 @@ public class TestBucketGet {
getBucket.setClient(client);
ListObjectResponse getBucketResponse =
getBucket.list("b1", "/", null, null, 100, "dir1", null);
(ListObjectResponse) getBucket
.list("b1", "/", null, null, 100, "dir1", null, null).getEntity();
Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size());
Assert.assertEquals("dir1/",
@ -87,7 +90,8 @@ public class TestBucketGet {
getBucket.setClient(ozoneClient);
ListObjectResponse getBucketResponse =
getBucket.list("b1", "/", null, null, 100, "dir1/", null);
(ListObjectResponse) getBucket
.list("b1", "/", null, null, 100, "dir1/", null, null).getEntity();
Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size());
Assert.assertEquals("dir1/dir2/",