This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new 1481a51 [ZEPPELIN-4896] Add S3 Canned ACL Option for S3NotebookRepo
1481a51 is described below
commit 1481a51fb191ad4cd191a4678d65912fe7578244
Author: Nathaniel Troutman <trou...@amazon.com>
AuthorDate: Sat Jun 20 00:08:59 2020 -0700
[ZEPPELIN-4896] Add S3 Canned ACL Option for S3NotebookRepo
### What is this PR for?
When S3NotebookRepo saves a notebook into S3 it does so without granting
the bucket owner full control of the object, this means that owning AWS account
cannot read the notebook, only the creating AWS account. This causes issues
when a notebook bucket is shared across AWS accounts.
This PR introduces the configuration value "zeppelin.notebook.s3.cannedAcl"
which
will override the default object permissions when saving a notebook to S3.
This allows
granting the bucket owner full control when the writer and the bucket owner
are
different AWS accounts.
### What type of PR is it?
Improvement/Feature
### Todos
* [ ] - N/A
### What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-4896
### How should this be tested?
#### Unit-Tests
Unfortunately Unit-Tests are not possible as gaul.S3Proxy does not support
Object ACL throwing a NotImplemented exception if you attempt to use
PutRequest.withCannedAcl for any value other than public or private (see
https://github.com/gaul/s3proxy#limitations).
#### Manual Testing
1. Have two AWS accounts, "account-a" and "account-b"
2. Create a bucket "notebook-bucket" in "account-a" and grant "account-b"
permissions to write to it
3. Setup zeppelin-site.xml for S3NotebookRepor WITHOUT Canned ACL Feature
and AWS Credentials for "account-b"
4. Launch zeppelin, create a new note titled 'WithoutACL', and shutdown
zeppelin
5. Verify that the permissions on the S3 Object do NOT grant the bucket
owner "account-a" permissions
```
aws s3api get-object-acl --bucket notebook-bucket --key
test-user/notebook/WithoutACL_2FD4NFYTU.zpln
{
"Owner": {
"DisplayName": "account-b",
"ID": "1e9...e4"
},
"Grants": [
{
"Grantee": {
"DisplayName": "account-b",
"ID": "1e9...e4",
"Type": "CanonicalUser"
},
"Permission": "FULL_CONTROL"
}
]
}
```
6. Edit zeppelin-site.xml enabling "zeppelin.notebook.s3.cannedAcl" as
"BucketOwnerFullControl"
7. Launch zeppelin, create a new note titled "WithACL", and shutdown
zeppelin
8. Verify that the permissions on the S3 object DO GRANT the bucket owner,
"account-a", full control
```
aws s3api get-object-acl --bucket notebook-bucket --key
test-user/notebook/WithACL_2FCXTUS3M.zpln
{
"Owner": {
"DisplayName": "account-b",
"ID": "1e9...e4"
},
"Grants": [
{
"Grantee": {
"DisplayName": "account-b",
"ID": "1e9...e4",
"Type": "CanonicalUser"
},
"Permission": "FULL_CONTROL"
},
{
"Grantee": {
"DisplayName": "account-a",
"ID": "f60...ee",
"Type": "CanonicalUser"
},
"Permission": "FULL_CONTROL"
}
]
}
```
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this need documentation? Yes, added relevant documentation along
side existing S3 Notebook Repo documentation.
Author: Nathaniel Troutman <trou...@amazon.com>
Closes #3811 from ntroutman/canned-acl and squashes the following commits:
6ebe608a1 [Nathaniel Troutman] [ZEPPELIN-4896] Add S3 Canned ACL Option for
S3NotebookRepo
(cherry picked from commit 5b3b819e7a127e95ea9a03ec841264e4d169f21a)
Signed-off-by: Jeff Zhang <zjf...@apache.org>
---
conf/zeppelin-site.xml.template | 9 +++++++++
docs/setup/operation/configuration.md | 6 ++++++
docs/setup/storage/storage.md | 21 +++++++++++++++++++++
.../apache/zeppelin/conf/ZeppelinConfiguration.java | 5 +++++
.../zeppelin/notebook/repo/OldS3NotebookRepo.java | 9 ++++++++-
.../zeppelin/notebook/repo/S3NotebookRepo.java | 8 ++++++++
.../src/interfaces/message-common.interface.ts | 1 +
7 files changed, 58 insertions(+), 1 deletion(-)
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index d355003..a40b9ec 100755
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -183,6 +183,15 @@
</property>
-->
+<!-- S3 Object Permissions (Canned ACL) for notebooks -->
+<!--
+<property>
+ <name>zeppelin.notebook.s3.cannedAcl</name>
+ <value>BucketOwnerFullControl</value>
+ <description>Saves notebooks in S3 with the given Canned Access Control
List.</description>
+</property>
+-->
+
<!-- Optional override to control which signature algorithm should be used to
sign AWS requests -->
<!-- Set this property to "S3SignerType" if your AWS S3 compatible APIs
support only AWS Signature Version 2 such as Ceph. -->
<!--
diff --git a/docs/setup/operation/configuration.md
b/docs/setup/operation/configuration.md
index 3c509b0..2a8b811 100644
--- a/docs/setup/operation/configuration.md
+++ b/docs/setup/operation/configuration.md
@@ -264,6 +264,12 @@ If both are defined, then the **environment variables**
will take priority.
<td>Save notebooks to S3 with server-side encryption enabled</td>
</tr>
<tr>
+ <td><h6 class="properties">ZEPPELIN_NOTEBOOK_S3_CANNED_ACL</h6></td>
+ <td><h6 class="properties">zeppelin.notebook.s3.cannedAcl</h6></td>
+ <td></td>
+ <td>Save notebooks to S3 with the given [Canned
ACL](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/CannedAccessControlList.html)
which determines the S3 permissions.</td>
+ </tr>
+ <tr>
<td><h6 class="properties">ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE</h6></td>
<td><h6 class="properties">zeppelin.notebook.s3.signerOverride</h6></td>
<td></td>
diff --git a/docs/setup/storage/storage.md b/docs/setup/storage/storage.md
index c1e0997..7867990 100644
--- a/docs/setup/storage/storage.md
+++ b/docs/setup/storage/storage.md
@@ -206,6 +206,27 @@ Or using the following setting in **zeppelin-site.xml**:
</br>
+### S3 Object Permissions
+
+S3 allows writing objects into buckets owned by a different account than the
requestor, when this happens S3 by default does not grant the bucket owner
permissions to the written object. Setting the Canned ACL when communicating
with S3 determines the permissions of notebooks saved in S3. Allowed values for
Canned ACL are found
[here](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/CannedAccessControlList.html),
the most frequent value is "BucketOwne [...]
+
+
+```bash
+export ZEPPELIN_NOTEBOOK_S3_CANNED_ACL=BucketOwnerFullControl
+```
+
+Or using the following setting in **zeppelin-site.xml**:
+
+```xml
+<property>
+ <name>zeppelin.notebook.s3.cannedAcl</name>
+ <value>BucketOwnerFullControl</value>
+ <description>Saves notebooks in S3 with the given Canned Access Control
List.</description>
+</property>
+```
+
+</br>
+
## Notebook Storage in Azure <a name="Azure"></a>
Using `AzureNotebookRepo` you can connect your Zeppelin with your Azure
account for notebook storage.
diff --git
a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 5637e1c..dcb8619 100644
---
a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -492,6 +492,10 @@ public class ZeppelinConfiguration extends
XMLConfiguration {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE);
}
+ public String getS3CannedAcl() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_CANNED_ACL);
+ }
+
public String getOSSBucketName() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_OSS_BUCKET);
}
@@ -943,6 +947,7 @@ public class ZeppelinConfiguration extends XMLConfiguration
{
ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion",
null),
ZEPPELIN_NOTEBOOK_S3_SSE("zeppelin.notebook.s3.sse", false),
ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE("zeppelin.notebook.s3.signerOverride",
null),
+ ZEPPELIN_NOTEBOOK_S3_CANNED_ACL("zeppelin.notebook.s3.cannedAcl", null),
ZEPPELIN_NOTEBOOK_OSS_BUCKET("zeppelin.notebook.oss.bucket", "zeppelin"),
ZEPPELIN_NOTEBOOK_OSS_ENDPOINT("zeppelin.notebook.oss.endpoint",
"http://oss-cn-hangzhou.aliyuncs.com";),
ZEPPELIN_NOTEBOOK_OSS_ACCESSKEYID("zeppelin.notebook.oss.accesskeyid",
null),
diff --git
a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/OldS3NotebookRepo.java
b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/OldS3NotebookRepo.java
index 5e4b653..42bcc6e 100644
---
a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/OldS3NotebookRepo.java
+++
b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/OldS3NotebookRepo.java
@@ -27,6 +27,7 @@ import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
+import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CryptoConfiguration;
import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
import com.amazonaws.services.s3.model.GetObjectRequest;
@@ -84,6 +85,7 @@ public class OldS3NotebookRepo implements OldNotebookRepo {
private String bucketName;
private String user;
private boolean useServerSideEncryption;
+ private CannedAccessControlList objectCannedAcl;
private ZeppelinConfiguration conf;
public OldS3NotebookRepo() {
@@ -95,6 +97,9 @@ public class OldS3NotebookRepo implements OldNotebookRepo {
bucketName = conf.getS3BucketName();
user = conf.getS3User();
useServerSideEncryption = conf.isS3ServerSideEncryption();
+ if (StringUtils.isNotBlank(conf.getS3CannedAcl())) {
+ objectCannedAcl = CannedAccessControlList.valueOf(conf.getS3CannedAcl());
+ }
// always use the default provider chain
AWSCredentialsProvider credentialsProvider = new
DefaultAWSCredentialsProviderChain();
@@ -244,7 +249,9 @@ public class OldS3NotebookRepo implements OldNotebookRepo {
objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
putRequest.setMetadata(objectMetadata);
}
-
+ if (objectCannedAcl != null) {
+ putRequest.withCannedAcl(objectCannedAcl);
+ }
s3client.putObject(putRequest);
}
catch (AmazonClientException ace) {
diff --git
a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
index 8df3149..d936b79 100644
---
a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
+++
b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
@@ -47,6 +47,7 @@ import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
+import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CryptoConfiguration;
import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
import com.amazonaws.services.s3.model.GetObjectRequest;
@@ -84,6 +85,7 @@ public class S3NotebookRepo implements NotebookRepo {
private String bucketName;
private String user;
private boolean useServerSideEncryption;
+ private CannedAccessControlList objectCannedAcl;
private ZeppelinConfiguration conf;
private String rootFolder;
@@ -97,6 +99,9 @@ public class S3NotebookRepo implements NotebookRepo {
user = conf.getS3User();
rootFolder = user + "/notebook";
useServerSideEncryption = conf.isS3ServerSideEncryption();
+ if (StringUtils.isNotBlank(conf.getS3CannedAcl())) {
+ objectCannedAcl =
CannedAccessControlList.valueOf(conf.getS3CannedAcl());
+ }
// always use the default provider chain
AWSCredentialsProvider credentialsProvider = new
DefaultAWSCredentialsProviderChain();
@@ -237,6 +242,9 @@ public class S3NotebookRepo implements NotebookRepo {
objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
putRequest.setMetadata(objectMetadata);
}
+ if (objectCannedAcl != null) {
+ putRequest.withCannedAcl(objectCannedAcl);
+ }
s3client.putObject(putRequest);
}
catch (AmazonClientException ace) {
diff --git
a/zeppelin-web-angular/projects/zeppelin-sdk/src/interfaces/message-common.interface.ts
b/zeppelin-web-angular/projects/zeppelin-sdk/src/interfaces/message-common.interface.ts
index 0a5ad6d..c0873c9 100644
---
a/zeppelin-web-angular/projects/zeppelin-sdk/src/interfaces/message-common.interface.ts
+++
b/zeppelin-web-angular/projects/zeppelin-sdk/src/interfaces/message-common.interface.ts
@@ -117,6 +117,7 @@ export interface ConfigurationsInfo {
'zeppelin.interpreter.localRepo': string;
'zeppelin.notebook.collaborative.mode.enable': string;
'zeppelin.search.use.disk': string;
+ 'zeppelin.notebook.s3.cannedAcl': string;
};
}
