Skip to content

Commit

Permalink
Add S3 access point test (#746)
Browse files Browse the repository at this point in the history 
* Add S3 access point test

* More virtual addressing fixes
  • Loading branch information
@adam-fowler
adam-fowler authored Dec 18, 2024
1 parent 3b62b62 commit d46f1bb
Show file tree
Hide file tree
Showing 2 changed files with 122 additions and 25 deletions.
4 changes: 2 additions & 2 deletions Tests/SotoTests/Services/S3/S3ExtensionTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -377,8 +377,8 @@ extension S3Tests {
"https://s3.us-east-1.amazonaws.com/bucket/file%20name",
s3URL: "https://bucket.s3.us-east-1.amazonaws.com/file%20name"
)
try await self.testS3VirtualAddressing("http://localhost:8000/bucket/filename", s3URL: "http://localhost:8000/bucket/filename")
try await self.testS3VirtualAddressing("http://localhost:8000/bucket//filename", s3URL: "http://localhost:8000/bucket//filename")
try await self.testS3VirtualAddressing("http://localhost:8000/bucket1/filename", s3URL: "http://localhost:8000/bucket1/filename")
try await self.testS3VirtualAddressing("http://localhost:8000/bucket2//filename", s3URL: "http://localhost:8000/bucket2//filename")
try await self.testS3VirtualAddressing("https://localhost:8000/bucket/file%20name", s3URL: "https://localhost:8000/bucket/file%20name")

let s3 = Self.s3.with(options: .s3ForceVirtualHost)
Expand Down
143 changes: 120 additions & 23 deletions Tests/SotoTests/Services/S3/S3Tests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@ import AsyncHTTPClient
import Atomics
import NIOCore
import NIOPosix
import SotoCore
import XCTest

@testable import SotoCore
@testable import SotoS3
@testable import SotoS3Control

Expand Down Expand Up @@ -68,7 +68,10 @@ class S3Tests: XCTestCase {
}

static func deleteBucket(name: String, s3: S3) async throws {
let response = try await s3.listObjectsV2(.init(bucket: name), logger: TestEnvironment.logger)
let response = try await s3.listObjectsV2(
.init(bucket: name),
logger: TestEnvironment.logger
)
if let contents = response.contents {
let request = S3.DeleteObjectsRequest(
bucket: name,
Expand All @@ -80,7 +83,13 @@ class S3Tests: XCTestCase {
}

/// create S3 bucket with supplied name and run supplied closure
func testBucket(_ name: String, s3: S3? = nil, test: @escaping (String) async throws -> Void) async throws {
func testBucket(
_ name: String,
s3: S3? = nil,
test: @escaping (String) async throws -> Void
)
async throws
{
let s3 = s3 ?? Self.s3!
try await XCTTestAsset {
try await Self.createBucket(name: name, s3: s3)
Expand All @@ -93,7 +102,14 @@ class S3Tests: XCTestCase {
}

/// Test putObject to S3 and that getObject returns the same object
func testPutGetObject(bucket: String, filename: String, contents: AWSHTTPBody, s3: S3? = nil) async throws {
func testPutGetObject(
bucket: String,
filename: String,
contents: AWSHTTPBody,
s3: S3? = nil
)
async throws
{
let s3 = s3 ?? Self.s3!
try await self.testBucket(bucket, s3: s3) { name in
let putRequest = S3.PutObjectRequest(
Expand All @@ -103,7 +119,9 @@ class S3Tests: XCTestCase {
)
let putResponse = try await s3.putObject(putRequest)
XCTAssertNotNil(putResponse.eTag)
let getResponse = try await s3.getObject(.init(bucket: name, key: filename, responseExpires: Date()))
let getResponse = try await s3.getObject(
.init(bucket: name, key: filename, responseExpires: Date())
)
let requestContents = try await contents.collect(upTo: .max)
let responseContents = try await getResponse.body.collect(upTo: .max)
XCTAssertEqual(responseContents, requestContents)
Expand Down Expand Up @@ -150,9 +168,15 @@ class S3Tests: XCTestCase {
)
let putResponse = try await Self.s3.putObject(putRequest)
XCTAssertNotNil(putResponse.eTag)
let copyRequest = S3.CopyObjectRequest(bucket: name, copySource: "\(name)/\(keyName)", key: newKeyName)
let copyRequest = S3.CopyObjectRequest(
bucket: name,
copySource: "\(name)/\(keyName)",
key: newKeyName
)
_ = try await Self.s3.copyObject(copyRequest)
let getResponse = try await Self.s3.getObject(.init(bucket: name, key: newKeyName, responseExpires: Date()))
let getResponse = try await Self.s3.getObject(
.init(bucket: name, key: newKeyName, responseExpires: Date())
)
let responseContents = try await getResponse.body.collect(upTo: .max)
XCTAssertEqual(String(buffer: responseContents), contents)
XCTAssertNotNil(getResponse.lastModified)
Expand Down Expand Up @@ -208,7 +232,11 @@ class S3Tests: XCTestCase {
let name = TestEnvironment.generateResourceName()
let contents = "testing S3.ListObjectsV2"
try await self.testBucket(name) { name in
let putRequest = S3.PutObjectRequest(body: .init(string: contents), bucket: name, key: name)
let putRequest = S3.PutObjectRequest(
body: .init(string: contents),
bucket: name,
key: name
)
let putResponse = try await Self.s3.putObject(putRequest)
let eTag = putResponse.eTag
let listResponse = try await Self.s3.listObjectsV2(.init(bucket: name))
Expand Down Expand Up @@ -269,7 +297,10 @@ class S3Tests: XCTestCase {
throw Disable100CompleteError(header: request.headers["Expect"].first)
}
}
let s3 = Self.s3.with(middleware: Disable100CompleteMiddleware(), options: .s3Disable100Continue)
let s3 = Self.s3.with(
middleware: Disable100CompleteMiddleware(),
options: .s3Disable100Continue
)
let name = TestEnvironment.generateResourceName()
let byteBuffer = Self.createRandomBuffer(size: 8 * 1024)

Expand All @@ -291,7 +322,8 @@ class S3Tests: XCTestCase {
let name = TestEnvironment.generateResourceName()
try await self.testBucket(name) { name in
// set lifecycle rules
let incompleteMultipartUploads = S3.AbortIncompleteMultipartUpload(daysAfterInitiation: 7) // clear incomplete multipart uploads after 7 days
// clear incomplete multipart uploads after 7 days
let incompleteMultipartUploads = S3.AbortIncompleteMultipartUpload(daysAfterInitiation: 7)
let filter = S3.LifecycleRuleFilter(prefix: "") // everything
let transitions = [S3.Transition(days: 14, storageClass: .glacier)] // transition objects to glacier after 14 days
let lifecycleRules = S3.LifecycleRule(
Expand All @@ -301,13 +333,19 @@ class S3Tests: XCTestCase {
status: .enabled,
transitions: transitions
)
let request = S3.PutBucketLifecycleConfigurationRequest(bucket: name, lifecycleConfiguration: .init(rules: [lifecycleRules]))
let request = S3.PutBucketLifecycleConfigurationRequest(
bucket: name,
lifecycleConfiguration: .init(rules: [lifecycleRules])
)
_ = try await Self.s3.putBucketLifecycleConfiguration(request)

let getResponse = try await Self.s3.getBucketLifecycleConfiguration(.init(bucket: name))
XCTAssertEqual(getResponse.rules?[0].transitions?[0].storageClass, .glacier)
XCTAssertEqual(getResponse.rules?[0].transitions?[0].days, 14)
XCTAssertEqual(getResponse.rules?[0].abortIncompleteMultipartUpload?.daysAfterInitiation, 7)
XCTAssertEqual(
getResponse.rules?[0].abortIncompleteMultipartUpload?.daysAfterInitiation,
7
)
}
}

Expand Down Expand Up @@ -343,7 +381,9 @@ class S3Tests: XCTestCase {
key: name
)
_ = try await Self.s3.putObject(putRequest)
let getACLResponse = try await Self.s3.getObjectAcl(.init(bucket: name, key: name, requestPayer: .requester))
let getACLResponse = try await Self.s3.getObjectAcl(
.init(bucket: name, key: name, requestPayer: .requester)
)
print(getACLResponse)
}
}
Expand All @@ -354,7 +394,9 @@ class S3Tests: XCTestCase {
try await _ = (0..<16).concurrentMap {
let body = "testMultipleUpload - " + $0.description
let filename = "file" + $0.description
_ = try await Self.s3.putObject(.init(body: .init(string: body), bucket: name, key: filename))
_ = try await Self.s3.putObject(
.init(body: .init(string: body), bucket: name, key: filename)
)
}

let paginator = Self.s3.listObjectsV2Paginator(.init(bucket: name, maxKeys: 5))
Expand Down Expand Up @@ -385,7 +427,10 @@ class S3Tests: XCTestCase {
try await self.testPutGetObject(
bucket: name,
filename: "testfile.txt",
contents: .init(asyncSequence: byteBuffer.asyncSequence(chunkSize: chunkSize), length: byteBuffer.readableBytes),
contents: .init(
asyncSequence: byteBuffer.asyncSequence(chunkSize: chunkSize),
length: byteBuffer.readableBytes
),
s3: s3
)
}
Expand Down Expand Up @@ -421,11 +466,17 @@ class S3Tests: XCTestCase {
try XCTSkipIf(TestEnvironment.isUsingLocalstack)

let name = TestEnvironment.generateResourceName()
let s3Url = URL(string: "https://\(name).s3.us-east-1.amazonaws.com/\(name)!=%25+/(*)_.txt")!
let s3Url = URL(
string: "https://\(name).s3.us-east-1.amazonaws.com/\(name)!=%25+/(*)_.txt"
)!

try await testBucket(name) { _ in
let byteBuffer = Self.createRandomBuffer(size: 186)
let putURL = try await Self.s3.signURL(url: s3Url, httpMethod: .PUT, expires: .minutes(5))
let putURL = try await Self.s3.signURL(
url: s3Url,
httpMethod: .PUT,
expires: .minutes(5)
)
var request = HTTPClientRequest(url: putURL.absoluteString)
request.method = .PUT
request.body = .bytes(byteBuffer)
Expand All @@ -435,8 +486,15 @@ class S3Tests: XCTestCase {
let listResponse = try await Self.s3.listObjectsV2(.init(bucket: name))
XCTAssertEqual(listResponse.contents?.first?.key, "\(name)!=%+/(*)_.txt")

let getURL = try await Self.s3.signURL(url: s3Url, httpMethod: .GET, expires: .minutes(5))
let getResponse = try await HTTPClient.shared.execute(.init(url: getURL.absoluteString), timeout: .minutes(1))
let getURL = try await Self.s3.signURL(
url: s3Url,
httpMethod: .GET,
expires: .minutes(5)
)
let getResponse = try await HTTPClient.shared.execute(
.init(url: getURL.absoluteString),
timeout: .minutes(1)
)

let getBuffer = try await getResponse.body.collect(upTo: .max)
XCTAssertEqual(response.status, .ok)
Expand Down Expand Up @@ -523,7 +581,10 @@ class S3Tests: XCTestCase {
let filename = "testfile.txt"
let contents = "testing S3.PutObject and S3.GetObject"
// set acceleration configuration
let request = S3.PutBucketAccelerateConfigurationRequest(accelerateConfiguration: .init(status: .enabled), bucket: name)
let request = S3.PutBucketAccelerateConfigurationRequest(
accelerateConfiguration: .init(status: .enabled),
bucket: name
)
_ = try await Self.s3.putBucketAccelerateConfiguration(request)

let putRequest = S3.PutObjectRequest(
Expand All @@ -533,7 +594,9 @@ class S3Tests: XCTestCase {
)
let putResponse = try await s3Accelerated.putObject(putRequest)
XCTAssertNotNil(putResponse.eTag)
let getResponse = try await s3Accelerated.getObject(.init(bucket: name, key: filename, responseExpires: Date()))
let getResponse = try await s3Accelerated.getObject(
.init(bucket: name, key: filename, responseExpires: Date())
)
let responseContents = try await getResponse.body.collect(upTo: .max)
XCTAssertEqual(String(buffer: responseContents), contents)
XCTAssertNotNil(getResponse.lastModified)
Expand All @@ -546,7 +609,9 @@ class S3Tests: XCTestCase {
let filename = "testfile.txt"
let contents = "testing S3.PutObject and S3.GetObject"

_ = try await Self.s3.putObject(.init(body: .init(string: contents), bucket: name, key: filename))
_ = try await Self.s3.putObject(
.init(body: .init(string: contents), bucket: name, key: filename)
)
try await Self.s3.waitUntilObjectExists(.init(bucket: name, key: filename))

_ = try await Self.s3.deleteObject(.init(bucket: name, key: filename))
Expand Down Expand Up @@ -577,7 +642,9 @@ class S3Tests: XCTestCase {
throw CancelError()
}
}
let s3Control = S3Control(client: Self.client, region: .euwest1).with(middleware: CheckHostMiddleware())
let s3Control = S3Control(client: Self.client, region: .euwest1).with(
middleware: CheckHostMiddleware()
)
let request = S3Control.ListJobsRequest(accountId: "123456780123")
do {
_ = try await s3Control.listJobs(request)
Expand Down Expand Up @@ -636,4 +703,34 @@ class S3Tests: XCTestCase {
}
}
}

/// test S3 control host is prefixed with account id
func testS3AccessPoints() async throws {
// doesnt work with LocalStack
try XCTSkipIf(TestEnvironment.isUsingLocalstack)
guard let accountId = Environment["AWS_ACCOUNT_ID"] else { throw XCTSkip() }
let name = TestEnvironment.generateResourceName()
try await self.testBucket(name) { name in
let s3Control = S3Control(client: Self.client, region: Self.s3.region)
let response = try await s3Control.createAccessPoint(
accountId: accountId,
bucket: name,
name: "test-accesspoint",
logger: TestEnvironment.logger
)
return try await withTeardown {
let accessPointArn = try XCTUnwrap(response.accessPointArn)
let alias = try XCTUnwrap(response.alias)
let string = "testing S3.PutObject and S3.GetObject"
// upload using arn
_ = try await Self.s3.putObject(body: .init(string: string), bucket: accessPointArn, key: name, logger: TestEnvironment.logger)
// download using alias
let getObjectResponse = try await Self.s3.getObject(bucket: alias, key: name, logger: TestEnvironment.logger)
let body = try await getObjectResponse.body.collect(upTo: .max)
XCTAssertEqual(String(buffer: body), string)
} teardown: {
try? await s3Control.deleteAccessPoint(accountId: accountId, name: "test-accesspoint", logger: TestEnvironment.logger)
}
}
}
}

0 comments on commit d46f1bb

Please sign in to comment.