Last active
June 10, 2021 15:10
-
-
Save whiteken/d593244822db17a5480d6cc5e662ac35 to your computer and use it in GitHub Desktop.
Read an AWS S3 file in memory without download. I needed content from an S3 stored file inside a Lambda function (which is read only). Regular Read-S3Object has to download the file first. If there is an easier way please let me know!
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| function Get-S3ObjectContent { | |
| <# | |
| .Synopsis | |
| Get-Content for AWS S3 file without download | |
| .DESCRIPTION | |
| Retreives file content from a file stored in Amazon S3 bucket without downloading | |
| .EXAMPLE | |
| Use the following example when already authenticated with aws, e.g. inside an aws lambda function | |
| Get-S3ObjectContent -Bucket 'my-bucket-name' -Key 'my-file.txt' -Region 'us-west-2' | |
| .EXAMPLE | |
| Pass aws secrets to authenticate with aws first. AWSPowerShell module must be is installed locally | |
| Get-S3ObjectContent -Bucket 'my-bucket-name' -Key 'my-file.txt' -Region 'us-west-2' -AccessKey <awsaccesskey> -SecretKey <awssecretkey> -ProfileName 'my-aws-profile-name' | |
| .NOTES | |
| https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html#RESTObjectSELECTContent-responses-examples | |
| #> | |
| [cmdletbinding()] | |
| [outputType([PSCustomObject])] | |
| param( | |
| #Supply the AWS S3 bucket name | |
| [Parameter(Mandatory=$true)][string]$Bucket, | |
| #Supply the key (filename/path) to the file | |
| [Parameter(Mandatory=$true)][string]$Key, | |
| #Supply the aws region | |
| [Parameter(Mandatory=$true)][string]$Region, | |
| #Supply aws access key | |
| [Parameter(ParameterSetName='Local', Mandatory=$true)][string]$AccessKey, | |
| #Supply aws secret key | |
| [Parameter(ParameterSetName='Local', Mandatory=$true)][string]$SecretKey, | |
| #Supply aws profile name | |
| [Parameter(ParameterSetName='Local', Mandatory=$true)][string]$ProfileName | |
| ) | |
| if($PSCmdlet.ParameterSetName -eq 'Local'){ | |
| if(-not(Get-Module AWSPowerShell)){ | |
| Import-Module AWSPowershell -ErrorAction Stop | |
| } | |
| if(-not(Get-AWSCredential)){ | |
| Set-AWSCredential -AccessKey $AccessKey -SecretKey $SecretKey -StoreAs $ProfileName | |
| Initialize-AWSDefaultConfiguration -ProfileName $ProfileName -Region $Region | |
| } | |
| } | |
| $iSerialization = [Amazon.S3.Model.InputSerialization]::new() | |
| $oSerialization = [Amazon.S3.Model.OutputSerialization]::new() | |
| $ijson = [Amazon.S3.Model.JSONInput]::new() | |
| $ijson.JsonType = 'DOCUMENT' | |
| $ojson = [Amazon.S3.Model.JSONOutput]::new() | |
| $ojson.RecordDelimiter = '\n' | |
| $iSerialization.json = $ijson | |
| $oSerialization.json = $ojson | |
| $params = @{ | |
| Bucket = $Bucket | |
| Key = $Key | |
| Expression = 'Select * from S3Object' | |
| ExpressionType = 'SQL' | |
| InputSerialization = $iserialization | |
| OutputSerialization = $oSerialization | |
| } | |
| $s3ObjectContent = Select-S3ObjectContent @params | |
| [System.IO.MemoryStream[]]$memoryStream = $s3ObjectContent.Payload | |
| $fileContent = foreach($result in $memoryStream){ | |
| $reader = [System.IO.StreamReader]::new($result, [System.Text.Encoding]::UTF8, $true) | |
| $result.Position = 0 | |
| $reader.ReadToEnd() | |
| } | |
| if($reader){ | |
| $reader.Dispose() | |
| } | |
| $json = $fileContent -replace '\\n' | |
| $json | ConvertFrom-Json | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
