Add aws s3 blob provider

pull/4826/head
liangshiwei 5 years ago
parent 56fc1797a7
commit 643ab8d526

@ -319,6 +319,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Volo.Abp.BlobStoring.Aliyun
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Volo.Abp.BlobStoring.Aliyun.Tests", "test\Volo.Abp.BlobStoring.Aliyun.Tests\Volo.Abp.BlobStoring.Aliyun.Tests.csproj", "{8E49687A-E69F-49F2-8DB0-428D0883A937}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Volo.Abp.BlobStoring.Aws", "src\Volo.Abp.BlobStoring.Aws\Volo.Abp.BlobStoring.Aws.csproj", "{50968CDE-1029-4051-B2E5-B69D0ECF2A18}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -949,6 +951,10 @@ Global
{8E49687A-E69F-49F2-8DB0-428D0883A937}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8E49687A-E69F-49F2-8DB0-428D0883A937}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8E49687A-E69F-49F2-8DB0-428D0883A937}.Release|Any CPU.Build.0 = Release|Any CPU
{50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Debug|Any CPU.Build.0 = Debug|Any CPU
{50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Release|Any CPU.ActiveCfg = Release|Any CPU
{50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -1110,6 +1116,7 @@ Global
{60D0E384-965E-4F81-9D71-B28F419254FC} = {447C8A77-E5F0-4538-8687-7383196D04EA}
{845E6A13-D1B5-4DDC-A16C-68D807E3B4C7} = {5DF0E140-0513-4D0D-BE2E-3D4D85CD70E6}
{8E49687A-E69F-49F2-8DB0-428D0883A937} = {447C8A77-E5F0-4538-8687-7383196D04EA}
{50968CDE-1029-4051-B2E5-B69D0ECF2A18} = {5DF0E140-0513-4D0D-BE2E-3D4D85CD70E6}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {BB97ECF4-9A84-433F-A80B-2A3285BDD1D5}

@ -0,0 +1,3 @@
<Weavers xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="FodyWeavers.xsd">
<ConfigureAwait ContinueOnCapturedContext="false" />
</Weavers>

@ -0,0 +1,30 @@
<?xml version="1.0" encoding="utf-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
<!-- This file was generated by Fody. Manual changes to this file will be lost when your project is rebuilt. -->
<xs:element name="Weavers">
<xs:complexType>
<xs:all>
<xs:element name="ConfigureAwait" minOccurs="0" maxOccurs="1">
<xs:complexType>
<xs:attribute name="ContinueOnCapturedContext" type="xs:boolean" />
</xs:complexType>
</xs:element>
</xs:all>
<xs:attribute name="VerifyAssembly" type="xs:boolean">
<xs:annotation>
<xs:documentation>'true' to run assembly verification (PEVerify) on the target assembly after all weavers have been executed.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="VerifyIgnoreCodes" type="xs:string">
<xs:annotation>
<xs:documentation>A comma-separated list of error codes that can be safely ignored in assembly verification.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="GenerateXsd" type="xs:boolean">
<xs:annotation>
<xs:documentation>'false' to turn off automatic generation of the XML Schema file.</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
</xs:element>
</xs:schema>

@ -0,0 +1,24 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="..\..\..\configureawait.props" />
<Import Project="..\..\..\common.props" />
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<GenerateAssemblyConfigurationAttribute>false</GenerateAssemblyConfigurationAttribute>
<GenerateAssemblyCompanyAttribute>false</GenerateAssemblyCompanyAttribute>
<GenerateAssemblyProductAttribute>false</GenerateAssemblyProductAttribute>
<RootNamespace />
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Volo.Abp.BlobStoring\Volo.Abp.BlobStoring.csproj" />
<ProjectReference Include="..\Volo.Abp.Caching\Volo.Abp.Caching.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.3.111.27" />
<PackageReference Include="AWSSDK.SecurityToken" Version="3.3.105.30" />
</ItemGroup>
</Project>

@ -0,0 +1,10 @@
using Volo.Abp.Modularity;
namespace Volo.Abp.BlobStoring.Azure
{
[DependsOn(typeof(AbpBlobStoringModule))]
public class AbpBlobStoringAzureModule : AbpModule
{
}
}

@ -0,0 +1,30 @@
using System;
using Volo.Abp.Caching;
namespace Volo.Abp.BlobStoring.Aws
{
[Serializable]
[CacheName("TemporaryCredentials")]
public class TemporaryCredentialsCacheItem
{
public const string Key = "AwsTemporaryCredentialsCache";
public string AccessKeyId { get; set; }
public string SecretAccessKey { get; set; }
public string SessionToken { get; set; }
public TemporaryCredentialsCacheItem()
{
}
public TemporaryCredentialsCacheItem(string accessKeyId,string secretAccessKey,string sessionToken)
{
AccessKeyId = accessKeyId;
SecretAccessKey = secretAccessKey;
SessionToken = sessionToken;
}
}
}

@ -0,0 +1,25 @@
using System;
namespace Volo.Abp.BlobStoring.Aws
{
public static class AwsBlobContainerConfigurationExtensions
{
public static AwsBlobProviderConfiguration GetAwsConfiguration(
this BlobContainerConfiguration containerConfiguration)
{
return new AwsBlobProviderConfiguration(containerConfiguration);
}
public static BlobContainerConfiguration UseAws(
this BlobContainerConfiguration containerConfiguration,
Action<AwsBlobProviderConfiguration> awsConfigureAction)
{
containerConfiguration.ProviderType = typeof(AwsBlobProvider);
containerConfiguration.NamingNormalizers.TryAdd<AwsBlobNamingNormalizer>();
awsConfigureAction(new AwsBlobProviderConfiguration(containerConfiguration));
return containerConfiguration;
}
}
}

@ -0,0 +1,52 @@
using System.Text.RegularExpressions;
using Volo.Abp.DependencyInjection;
namespace Volo.Abp.BlobStoring.Aws
{
public class AwsBlobNamingNormalizer : IBlobNamingNormalizer, ITransientDependency
{
/// <summary>
///https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
/// </summary>
public virtual string NormalizeContainerName(string containerName)
{
// All letters in a container name must be lowercase.
containerName = containerName.ToLower();
// Container names can contain only letters, numbers, and the dash (-) character.
containerName = Regex.Replace(containerName, "[^a-z0-9-]", string.Empty);
// Every dash (-) character must be immediately preceded and followed by a letter or number;
// consecutive dashes are not permitted in container names.
// Container names must start or end with a letter or number
containerName = Regex.Replace(containerName, "-{2,}", "-");
containerName = Regex.Replace(containerName, "^-", string.Empty);
containerName = Regex.Replace(containerName, "-$", string.Empty);
// Container names must be from 3 through 63 characters long.
if (containerName.Length < 3)
{
var length = containerName.Length;
for (var i = 0; i < 3 - length; i++)
{
containerName += "0";
}
}
if (containerName.Length > 63)
{
containerName = containerName.Substring(0, 63);
}
return containerName;
}
/// <summary>
/// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
/// </summary>
public virtual string NormalizeBlobName(string blobName)
{
return blobName;
}
}
}

@ -0,0 +1,148 @@
using System;
using System.IO;
using System.Threading.Tasks;
using Amazon.S3;
using Amazon.S3.Model;
using Amazon.S3.Util;
using Volo.Abp.DependencyInjection;
namespace Volo.Abp.BlobStoring.Aws
{
public class AwsBlobProvider : BlobProviderBase, ITransientDependency
{
protected IAwsBlobNameCalculator AwsBlobNameCalculator { get; }
protected IAmazonS3ClientFactory AmazonS3ClientFactory { get; }
public AwsBlobProvider(IAwsBlobNameCalculator awsBlobNameCalculator,
IAmazonS3ClientFactory amazonS3ClientFactory)
{
AwsBlobNameCalculator = awsBlobNameCalculator;
AmazonS3ClientFactory = amazonS3ClientFactory;
}
public override async Task SaveAsync(BlobProviderSaveArgs args)
{
var blobName = AwsBlobNameCalculator.Calculate(args);
var configuration = args.Configuration.GetAwsConfiguration();
var containerName = GetContainerName(args);
using (var amazonS3Client = await GetAmazonS3Client(args))
{
if (!args.OverrideExisting && await BlobExistsAsync(amazonS3Client, containerName, blobName))
{
throw new BlobAlreadyExistsException(
$"Saving BLOB '{args.BlobName}' does already exists in the container '{containerName}'! Set {nameof(args.OverrideExisting)} if it should be overwritten.");
}
if (configuration.CreateContainerIfNotExists)
{
await amazonS3Client.PutBucketAsync(containerName);
}
await amazonS3Client.PutObjectAsync(new PutObjectRequest
{
BucketName = containerName,
Key = blobName,
InputStream = args.BlobStream
});
}
}
public override async Task<bool> DeleteAsync(BlobProviderDeleteArgs args)
{
var blobName = AwsBlobNameCalculator.Calculate(args);
var containerName = GetContainerName(args);
using (var amazonS3Client = await GetAmazonS3Client(args))
{
if (!await BlobExistsAsync(amazonS3Client, containerName, blobName))
{
return false;
}
await amazonS3Client.DeleteObjectAsync(new DeleteObjectRequest
{
BucketName = containerName,
Key = blobName
});
return true;
}
}
public override async Task<bool> ExistsAsync(BlobProviderExistsArgs args)
{
var blobName = AwsBlobNameCalculator.Calculate(args);
var containerName = GetContainerName(args);
using (var amazonS3Client = await GetAmazonS3Client(args))
{
return await BlobExistsAsync(amazonS3Client, containerName, blobName);
}
}
public override async Task<Stream> GetOrNullAsync(BlobProviderGetArgs args)
{
var blobName = AwsBlobNameCalculator.Calculate(args);
var containerName = GetContainerName(args);
using (var amazonS3Client = await GetAmazonS3Client(args))
{
if (!await BlobExistsAsync(amazonS3Client, containerName, blobName))
{
return null;
}
var response = await amazonS3Client.GetObjectAsync(new GetObjectRequest
{
BucketName = containerName,
Key = blobName
});
var memoryStream = new MemoryStream();
await response.ResponseStream.CopyToAsync(memoryStream);
return memoryStream;
}
}
protected virtual async Task<AmazonS3Client> GetAmazonS3Client(BlobProviderArgs args)
{
var configuration = args.Configuration.GetAwsConfiguration();
return await AmazonS3ClientFactory.GetAmazonS3Client(configuration);
}
private async Task<bool> BlobExistsAsync(AmazonS3Client amazonS3Client, string containerName, string blobName)
{
// Make sure Blob Container exists.
if (!await AmazonS3Util.DoesS3BucketExistV2Async(amazonS3Client, containerName))
{
return false;
}
try
{
await amazonS3Client.GetObjectMetadataAsync(containerName, blobName);
}
catch (Exception ex)
{
if (ex is AmazonS3Exception)
{
return false;
}
throw;
}
return true;
}
private static string GetContainerName(BlobProviderArgs args)
{
var configuration = args.Configuration.GetAwsConfiguration();
return configuration.ContainerName.IsNullOrWhiteSpace()
? args.ContainerName
: configuration.ContainerName;
}
}
}

@ -0,0 +1,107 @@
using Amazon;
using Amazon.Runtime;
using Amazon.Runtime.CredentialManagement;
using Amazon.S3;
namespace Volo.Abp.BlobStoring.Aws
{
public class AwsBlobProviderConfiguration
{
public string AccessKeyId
{
get => _containerConfiguration.GetConfiguration<string>(AwsBlobProviderConfigurationNames.AccessKeyId);
set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.AccessKeyId, value);
}
public string SecretAccessKey
{
get => _containerConfiguration.GetConfiguration<string>(AwsBlobProviderConfigurationNames.SecretAccessKey);
set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.SecretAccessKey, value);
}
public bool UseAwsCredentials
{
get => _containerConfiguration.GetConfiguration<bool>(AwsBlobProviderConfigurationNames.UseAwsCredentials);
set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.UseAwsCredentials, value);
}
public bool UseTemporaryCredentials
{
get => _containerConfiguration.GetConfiguration<bool>(AwsBlobProviderConfigurationNames.UseTemporaryCredentials);
set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.UseTemporaryCredentials, value);
}
public bool UseTemporaryFederatedCredentials
{
get => _containerConfiguration.GetConfiguration<bool>(AwsBlobProviderConfigurationNames.UseTemporaryFederatedCredentials);
set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.UseTemporaryFederatedCredentials, value);
}
public string ProfileName
{
get => _containerConfiguration.GetConfiguration<string>(AwsBlobProviderConfigurationNames.ProfileName);
set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.ProfileName, value);
}
public string ProfilesLocation
{
get => _containerConfiguration.GetConfiguration<string>(AwsBlobProviderConfigurationNames.ProfilesLocation);
set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.ProfilesLocation, value);
}
/// <summary>
/// Set the validity period of the temporary access credential, the unit is s, the minimum is 900, and the maximum is 129600.
/// </summary>
public int DurationSeconds
{
get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.DurationSeconds, 0);
set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.DurationSeconds, value);
}
public string Name
{
get => _containerConfiguration.GetConfiguration<string>(AwsBlobProviderConfigurationNames.Name);
set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Name, value);
}
public string Policy
{
get => _containerConfiguration.GetConfiguration<string>(AwsBlobProviderConfigurationNames.Policy);
set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Policy, value);
}
public RegionEndpoint Region
{
get => _containerConfiguration.GetConfiguration<RegionEndpoint>(AwsBlobProviderConfigurationNames.Region);
set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Region, Check.NotNull(value, nameof(value)));
}
/// <summary>
/// This name may only contain lowercase letters, numbers, and hyphens, and must begin with a letter or a number.
/// Each hyphen must be preceded and followed by a non-hyphen character.
/// The name must also be between 3 and 63 characters long.
/// If this parameter is not specified, the ContainerName of the <see cref="BlobProviderArgs"/> will be used.
/// </summary>
public string ContainerName
{
get => _containerConfiguration.GetConfiguration<string>(AwsBlobProviderConfigurationNames.ContainerName);
set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.ContainerName, Check.NotNullOrWhiteSpace(value, nameof(value)));
}
/// <summary>
/// Default value: false.
/// </summary>
public bool CreateContainerIfNotExists
{
get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.CreateContainerIfNotExists, false);
set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.CreateContainerIfNotExists, value);
}
private readonly BlobContainerConfiguration _containerConfiguration;
public AwsBlobProviderConfiguration(BlobContainerConfiguration containerConfiguration)
{
_containerConfiguration = containerConfiguration;
}
}
}

@ -0,0 +1,19 @@
namespace Volo.Abp.BlobStoring.Aws
{
public static class AwsBlobProviderConfigurationNames
{
public const string AccessKeyId = "Aws.AccessKeyId";
public const string SecretAccessKey = "Aws.SecretAccessKey";
public const string UseAwsCredentials = "Aws.UseAWSCredentials";
public const string UseTemporaryCredentials = "Aws.UseTemporaryCredentials";
public const string UseTemporaryFederatedCredentials = "Aws.UseTemporaryFederatedCredentials";
public const string ProfileName = "Aws.ProfileName";
public const string ProfilesLocation = "Aws.ProfilesLocation";
public const string DurationSeconds = "Aws.DurationSeconds";
public const string Name = "Aws.Name";
public const string Policy = "Aws.Policy";
public const string Region = "Aws.Region";
public const string ContainerName = "Aws.ContainerName";
public const string CreateContainerIfNotExists = "Aws.CreateContainerIfNotExists";
}
}

@ -0,0 +1,168 @@
using System;
using System.Threading.Tasks;
using Amazon.Runtime;
using Amazon.Runtime.CredentialManagement;
using Amazon.S3;
using Amazon.SecurityToken;
using Amazon.SecurityToken.Model;
using Microsoft.Extensions.Caching.Distributed;
using Volo.Abp.Caching;
using Volo.Abp.DependencyInjection;
namespace Volo.Abp.BlobStoring.Aws
{
public class DefaultAmazonS3ClientFactory : IAmazonS3ClientFactory, ITransientDependency
{
protected IDistributedCache<TemporaryCredentialsCacheItem> Cache { get; }
public DefaultAmazonS3ClientFactory(IDistributedCache<TemporaryCredentialsCacheItem> cache)
{
Cache = cache;
}
public virtual async Task<AmazonS3Client> GetAmazonS3Client(
AwsBlobProviderConfiguration configuration)
{
if (configuration.UseAwsCredentials)
{
return new AmazonS3Client(GetAwsCredentials(configuration), configuration.Region);
}
if (configuration.UseTemporaryCredentials)
{
return new AmazonS3Client(await GetTemporaryCredentialsAsync(configuration), configuration.Region);
}
if (configuration.UseTemporaryFederatedCredentials)
{
return new AmazonS3Client(await GetTemporaryFederatedCredentialsAsync(configuration),
configuration.Region);
}
Check.NotNullOrWhiteSpace(configuration.AccessKeyId, nameof(configuration.AccessKeyId));
Check.NotNullOrWhiteSpace(configuration.SecretAccessKey, nameof(configuration.SecretAccessKey));
return new AmazonS3Client(configuration.AccessKeyId, configuration.SecretAccessKey);
}
protected virtual AWSCredentials GetAwsCredentials(
AwsBlobProviderConfiguration configuration)
{
var chain = new CredentialProfileStoreChain(configuration.ProfilesLocation);
if (chain.TryGetAWSCredentials(configuration.ProfileName, out var awsCredentials))
{
return awsCredentials;
}
throw new AmazonS3Exception("Not found aws credentials");
}
protected virtual async Task<SessionAWSCredentials> GetTemporaryCredentialsAsync(
AwsBlobProviderConfiguration configuration)
{
var temporaryCredentialsCache = await Cache.GetAsync(TemporaryCredentialsCacheItem.Key);
if (temporaryCredentialsCache == null)
{
AmazonSecurityTokenServiceClient stsClient;
if (!configuration.AccessKeyId.IsNullOrEmpty() && !configuration.SecretAccessKey.IsNullOrEmpty())
{
stsClient = new AmazonSecurityTokenServiceClient(configuration.AccessKeyId,
configuration.SecretAccessKey);
}
else
{
stsClient = new AmazonSecurityTokenServiceClient(GetAwsCredentials(configuration));
}
using (stsClient)
{
var getSessionTokenRequest = new GetSessionTokenRequest
{
DurationSeconds = configuration.DurationSeconds
};
var sessionTokenResponse =
await stsClient.GetSessionTokenAsync(getSessionTokenRequest);
var credentials = sessionTokenResponse.Credentials;
temporaryCredentialsCache =
await SetTemporaryCredentialsCache(credentials, configuration.DurationSeconds);
}
}
var sessionCredentials = new SessionAWSCredentials(
temporaryCredentialsCache.AccessKeyId,
temporaryCredentialsCache.SecretAccessKey,
temporaryCredentialsCache.SessionToken);
return sessionCredentials;
}
protected virtual async Task<SessionAWSCredentials> GetTemporaryFederatedCredentialsAsync(
AwsBlobProviderConfiguration configuration)
{
Check.NotNullOrWhiteSpace(configuration.Name, nameof(configuration.Name));
var temporaryCredentialsCache = await Cache.GetAsync(TemporaryCredentialsCacheItem.Key);
if (temporaryCredentialsCache == null)
{
AmazonSecurityTokenServiceClient stsClient;
if (!configuration.AccessKeyId.IsNullOrEmpty() && !configuration.SecretAccessKey.IsNullOrEmpty())
{
stsClient = new AmazonSecurityTokenServiceClient(configuration.AccessKeyId,
configuration.SecretAccessKey);
}
else
{
stsClient = new AmazonSecurityTokenServiceClient(GetAwsCredentials(configuration));
}
using (stsClient)
{
var federationTokenRequest =
new GetFederationTokenRequest
{
DurationSeconds = configuration.DurationSeconds,
Name = configuration.Name,
Policy = configuration.Policy
};
var federationTokenResponse =
await stsClient.GetFederationTokenAsync(federationTokenRequest);
var credentials = federationTokenResponse.Credentials;
temporaryCredentialsCache =
await SetTemporaryCredentialsCache(credentials, configuration.DurationSeconds);
}
}
var sessionCredentials = new SessionAWSCredentials(
temporaryCredentialsCache.AccessKeyId,
temporaryCredentialsCache.SecretAccessKey,
temporaryCredentialsCache.SessionToken);
return sessionCredentials;
}
private async Task<TemporaryCredentialsCacheItem> SetTemporaryCredentialsCache(
Credentials credentials,
int durationSeconds)
{
var temporaryCredentialsCache = new TemporaryCredentialsCacheItem(credentials.AccessKeyId,
credentials.SecretAccessKey,
credentials.SessionToken);
await Cache.SetAsync(TemporaryCredentialsCacheItem.Key, temporaryCredentialsCache,
new DistributedCacheEntryOptions
{
AbsoluteExpirationRelativeToNow = TimeSpan.FromSeconds(durationSeconds - 10)
});
return temporaryCredentialsCache;
}
}
}

@ -0,0 +1,10 @@
using System.Threading.Tasks;
using Amazon.S3;
namespace Volo.Abp.BlobStoring.Aws
{
public interface IAmazonS3ClientFactory
{
Task<AmazonS3Client> GetAmazonS3Client(AwsBlobProviderConfiguration configuration);
}
}

@ -0,0 +1,7 @@
namespace Volo.Abp.BlobStoring.Aws
{
public interface IAwsBlobNameCalculator
{
string Calculate(BlobProviderArgs args);
}
}
Loading…
Cancel
Save