@@ -12,69 +12,6 @@ import {
1212import { LogLevel } from '../types/apillon' ;
1313import { randomBytes } from 'crypto' ;
1414
15- function listFilesRecursive (
16- folderPath : string ,
17- fileList = [ ] ,
18- relativePath = '' ,
19- ) {
20- const gitignorePath = path . join ( folderPath , '.gitignore' ) ;
21- const gitignorePatterns = fs . existsSync ( gitignorePath )
22- ? fs . readFileSync ( gitignorePath , 'utf-8' ) . split ( '\n' )
23- : [ ] ;
24- gitignorePatterns . push ( '.git' ) ; // Always ignore .git folder.
25-
26- const files = fs . readdirSync ( folderPath ) ;
27- for ( const file of files ) {
28- const fullPath = path . join ( folderPath , file ) ;
29- const relativeFilePath = path . join ( relativePath , file ) ;
30-
31- // Skip file if it matches .gitignore patterns
32- if (
33- gitignorePatterns . some ( ( pattern ) =>
34- new RegExp ( pattern ) . test ( relativeFilePath ) ,
35- )
36- ) {
37- continue ;
38- }
39-
40- if ( fs . statSync ( fullPath ) . isDirectory ( ) ) {
41- listFilesRecursive ( fullPath , fileList , `${ relativeFilePath } ) ;
42- } else {
43- fileList . push ( { fileName : file , path : relativePath , index : fullPath } ) ;
44- }
45- }
46- return fileList . sort ( ( a , b ) => a . fileName . localeCompare ( b . fileName ) ) ;
47- }
48-
49- async function uploadFilesToS3 (
50- uploadLinks : ( FileMetadata & { url ?: string } ) [ ] ,
51- files : ( FileMetadata & { index ?: string } ) [ ] ,
52- ) {
53- const s3Api = axios . create ( ) ;
54- const uploadWorkers = [ ] ;
55-
56- for ( const link of uploadLinks ) {
57- // console.log(link.url);
58- const file = files . find (
59- ( x ) => x . fileName === link . fileName && ( ! x . path || x . path === link . path ) ,
60- ) ;
61- if ( ! file ) {
62- throw new Error ( `Can't find file ${ link . path } ${ link . fileName } ) ;
63- }
64- uploadWorkers . push (
65- new Promise < void > ( async ( resolve , _reject ) => {
66- // If uploading from local folder then read file, otherwise directly upload content
67- const content = file . index ? fs . readFileSync ( file . index ) : file . content ;
68- await s3Api . put ( link . url , content ) ;
69- ApillonLogger . log ( `File uploaded: ${ file . fileName } ) ;
70- resolve ( ) ;
71- } ) ,
72- ) ;
73- }
74-
75- await Promise . all ( uploadWorkers ) ;
76- }
77-
7815export async function uploadFiles (
7916 folderPath : string ,
8017 apiPrefix : string ,
@@ -88,10 +25,11 @@ export async function uploadFiles(
8825 } else {
8926 throw new Error ( 'Invalid upload parameters received' ) ;
9027 }
28+
9129 // If folderPath param passed, read files from local storage
9230 if ( folderPath && ! files ?. length ) {
9331 try {
94- files = listFilesRecursive ( folderPath ) ;
32+ files = readFilesFromFolder ( folderPath , params ?. ignoreFiles ) ;
9533 } catch ( err ) {
9634 ApillonLogger . log ( err . message , LogLevel . ERROR ) ;
9735 throw new Error ( `Error reading files in ${ folderPath } ) ;
@@ -124,6 +62,90 @@ export async function uploadFiles(
12462 return { sessionUuid, files : uploadedFiles . flatMap ( ( f ) => f ) } ;
12563}
12664
65+ function readFilesFromFolder (
66+ folderPath : string ,
67+ ignoreFiles = true ,
68+ ) : FileMetadata [ ] {
69+ const gitignorePatterns = [ ] ;
70+ if ( ignoreFiles ) {
71+ ApillonLogger . log ( 'Ignoring files from .gitignore during upload.' ) ;
72+
73+ const gitignorePath = path . join ( folderPath , '.gitignore' ) ;
74+ if ( fs . existsSync ( gitignorePath ) ) {
75+ gitignorePatterns . push (
76+ ...fs . readFileSync ( gitignorePath , 'utf-8' ) . split ( '\n' ) ,
77+ ) ;
78+ }
79+ // Ignore the following files by default when ignoreFiles = true
80+ gitignorePatterns . push (
81+ '\\.git/?$' ,
82+ '\\.gitignore$' ,
83+ 'node_modules/?' ,
84+ '\\.env$' ,
85+ ) ;
86+ }
87+
88+ const folderFiles = listFilesRecursive ( folderPath ) ;
89+ return folderFiles . filter (
90+ ( file ) =>
91+ // Skip files that match .gitignore patterns
92+ ! gitignorePatterns . some (
93+ ( pattern ) =>
94+ new RegExp ( pattern ) . test ( file . fileName ) ||
95+ new RegExp ( pattern ) . test ( file . path ) ,
96+ ) ,
97+ ) ;
98+ }
99+
100+ function listFilesRecursive (
101+ folderPath : string ,
102+ fileList = [ ] ,
103+ relativePath = '' ,
104+ ) : FileMetadata [ ] {
105+ const files = fs . readdirSync ( folderPath ) ;
106+
107+ for ( const file of files ) {
108+ const fullPath = path . join ( folderPath , file ) ;
109+ const relativeFilePath = path . join ( relativePath , file ) ;
110+
111+ if ( fs . statSync ( fullPath ) . isDirectory ( ) ) {
112+ listFilesRecursive ( fullPath , fileList , `${ relativeFilePath } ) ;
113+ } else {
114+ fileList . push ( { fileName : file , path : relativePath , index : fullPath } ) ;
115+ }
116+ }
117+ return fileList . sort ( ( a , b ) => a . fileName . localeCompare ( b . fileName ) ) ;
118+ }
119+
120+ async function uploadFilesToS3 (
121+ uploadLinks : ( FileMetadata & { url ?: string } ) [ ] ,
122+ files : ( FileMetadata & { index ?: string } ) [ ] ,
123+ ) {
124+ const s3Api = axios . create ( ) ;
125+ const uploadWorkers = [ ] ;
126+
127+ for ( const link of uploadLinks ) {
128+ // console.log(link.url);
129+ const file = files . find (
130+ ( x ) => x . fileName === link . fileName && ( ! x . path || x . path === link . path ) ,
131+ ) ;
132+ if ( ! file ) {
133+ throw new Error ( `Can't find file ${ link . path } ${ link . fileName } ) ;
134+ }
135+ uploadWorkers . push (
136+ new Promise < void > ( async ( resolve , _reject ) => {
137+ // If uploading from local folder then read file, otherwise directly upload content
138+ const content = file . index ? fs . readFileSync ( file . index ) : file . content ;
139+ await s3Api . put ( link . url , content ) ;
140+ ApillonLogger . log ( `File uploaded: ${ file . fileName } ) ;
141+ resolve ( ) ;
142+ } ) ,
143+ ) ;
144+ }
145+
146+ await Promise . all ( uploadWorkers ) ;
147+ }
148+
127149function chunkify ( files : FileMetadata [ ] , chunkSize = 10 ) : FileMetadata [ ] [ ] {
128150 // Divide files into chunks for parallel processing and uploading
129151 const fileChunks : FileMetadata [ ] [ ] = [ ] ;
0 commit comments