Documentation
¶
Index ¶
- func BatchUpload(ctx context.Context, g3i client.Gen3Interface, ...)
- func CompleteMultipartUpload(ctx context.Context, g3 client.Gen3Interface, key string, uploadID string, ...) error
- func FormatSize(size int64) string
- func GetWaitTime(retryCount int) time.Duration
- func InitBatchUploadChannels(numParallel int, inputSliceLen int) (int, chan *http.Response, chan error, []common.FileUploadRequestObject)
- func MultipartUpload(ctx context.Context, g3 client.Gen3Interface, ...) error
- func OptimalChunkSize(fileSize int64) int64
- func ProcessFilename(logger *logs.Gen3Logger, uploadPath string, filePath string, objectId string, ...) (common.FileUploadRequestObject, error)
- func RegisterAndUploadFile(ctx context.Context, g3 client.Gen3Interface, drsObject *drs.DRSObject, ...) (*drs.DRSObject, error)
- func RetryFailedUploads(ctx context.Context, g3 client.Gen3Interface, ...)
- func SeparateSingleAndMultipartUploads(g3i client.Gen3Interface, objects []common.FileUploadRequestObject) ([]common.FileUploadRequestObject, []common.FileUploadRequestObject)
- func Upload(ctx context.Context, g3 client.Gen3Interface, ...) error
- func UploadSingle(ctx context.Context, g3Client client.Gen3Interface, ...) error
- func UploadSingleFile(ctx context.Context, g3 client.Gen3Interface, ...) error
- type FileInfo
- type PresignedURLResponse
- type RenamedOrSkippedFileInfo
- type ShepherdAuthz
- type ShepherdInitRequestObject
- type UploadConfig
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func BatchUpload ¶
func CompleteMultipartUpload ¶
func CompleteMultipartUpload(ctx context.Context, g3 client.Gen3Interface, key string, uploadID string, parts []fence.MultipartPart, bucketName string) error
func FormatSize ¶
FormatSize helps to parse a int64 size into string
func GetWaitTime ¶
GetWaitTime calculates exponential backoff with cap
func InitBatchUploadChannels ¶
func MultipartUpload ¶
func MultipartUpload(ctx context.Context, g3 client.Gen3Interface, req common.FileUploadRequestObject, file *os.File, showProgress bool) error
func OptimalChunkSize ¶
OptimalChunkSize returns a recommended chunk size for the given fileSize (in bytes). - <= 100 MB: return fileSize (use single PUT) - >100 MB and <= 1 GB: 10 MB - >1 GB and <= 10 GB: scaled between 25 MB and 128 MB - >10 GB and <= 100 GB: 256 MB - >100 GB: scaled between 512 MB and 1024 MB (1 GB) See: https://cloud.switch.ch/-/documentation/s3/multipart-uploads/#best-practices
func ProcessFilename ¶
func ProcessFilename(logger *logs.Gen3Logger, uploadPath string, filePath string, objectId string, includeSubDirName bool, includeMetadata bool) (common.FileUploadRequestObject, error)
ProcessFilename returns an FileInfo object which has the information about the path and name to be used for upload of a file
func RegisterAndUploadFile ¶
func RegisterAndUploadFile(ctx context.Context, g3 client.Gen3Interface, drsObject *drs.DRSObject, filePath string, bucketName string, upsert bool) (*drs.DRSObject, error)
RegisterAndUploadFile orchestrates registration with Indexd and uploading via Fence. It handles checking for existing records, upsert logic, checking if file is already downloadable, and performing the upload.
func RetryFailedUploads ¶
func RetryFailedUploads(ctx context.Context, g3 client.Gen3Interface, failedMap map[string]common.RetryObject)
RetryFailedUploads re-uploads previously failed files with exponential backoff
func SeparateSingleAndMultipartUploads ¶
func SeparateSingleAndMultipartUploads(g3i client.Gen3Interface, objects []common.FileUploadRequestObject) ([]common.FileUploadRequestObject, []common.FileUploadRequestObject)
func Upload ¶
func Upload(ctx context.Context, g3 client.Gen3Interface, req common.FileUploadRequestObject, showProgress bool) error
Upload is a unified catch-all function that automatically chooses between single-part and multipart upload based on file size.
func UploadSingle ¶
func UploadSingle(ctx context.Context, g3Client client.Gen3Interface, req common.FileUploadRequestObject, showProgress bool) error
func UploadSingleFile ¶
func UploadSingleFile(ctx context.Context, g3 client.Gen3Interface, req common.FileUploadRequestObject, showProgress bool) error
UploadSingleFile handles single-part upload with progress
Types ¶
type FileInfo ¶
type FileInfo struct {
FilePath string
Filename string
FileMetadata common.FileMetadata
ObjectId string
}
FileInfo is a helper struct for including subdirname as filename
type PresignedURLResponse ¶
func GeneratePresignedUploadURL ¶
func GeneratePresignedUploadURL(ctx context.Context, g3 client.Gen3Interface, filename string, metadata common.FileMetadata, bucket string) (*PresignedURLResponse, error)
GeneratePresignedURL handles both Shepherd and Fence fallback
type RenamedOrSkippedFileInfo ¶
RenamedOrSkippedFileInfo is a helper struct for recording renamed or skipped files
type ShepherdAuthz ¶
type ShepherdInitRequestObject ¶
type ShepherdInitRequestObject struct {
Filename string `json:"file_name"`
Authz ShepherdAuthz `json:"authz"`
Aliases []string `json:"aliases"`
// Metadata is an encoded JSON string of any arbitrary metadata the user wishes to upload.
Metadata map[string]any `json:"metadata"`
}
ShepherdInitRequestObject represents the payload that sends to Shepherd for getting a singlepart upload presignedURL or init a multipart upload for new object file
Source Files