Skip to content

Commit eb3df85

Browse files
ashmeenkaurashmeenkaur
committed
merge conflicts
1 parent e9f8fa9 commit eb3df85

File tree

5 files changed

+164
-206
lines changed

5 files changed

+164
-206
lines changed

‎tools/integration_tests/read_cache/cache_file_for_exclude_regex_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,8 @@ func TestCacheFileForExcludeRegexTest(t *testing.T) {
9999
flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name())
100100
if setup.OnlyDirMounted() != "" {
101101
flagsSet = append(flagsSet,
102-
[]string{fmt.Sprintf("--file-cache-experimental-exclude-regex=^%s/%s/", setup.TestBucket(), onlyDirMounted), "--file-cache-max-size-mb=50", "--file-cache-cache-file-for-range-read=true", "--file-cache-enable-parallel-downloads=false", "--file-cache-enable-o-direct=false", fmt.Sprintf("--cache-dir=%s/gcsfuse-tmp/TestCacheFileForExcludeRegexTest", setup.TestDir()), fmt.Sprintf("--log-file=%s/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log", setup.TestDir()), "--log-severity=TRACE"},
103-
[]string{fmt.Sprintf("--file-cache-experimental-exclude-regex=^%s/%s/", setup.TestBucket(), onlyDirMounted), "--file-cache-max-size-mb=50", "--file-cache-cache-file-for-range-read=true", "--file-cache-enable-parallel-downloads=false", "--file-cache-enable-o-direct=false", fmt.Sprintf("--cache-dir=%s/gcsfuse-tmp/TestCacheFileForExcludeRegexTest", setup.TestDir()), fmt.Sprintf("--log-file=%s/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log", setup.TestDir()), "--log-severity=TRACE", "--client-protocol=grpc"},
102+
[]string{fmt.Sprintf("--file-cache-exclude-regex=^%s/%s/", setup.TestBucket(), onlyDirMounted), "--file-cache-max-size-mb=50", "--file-cache-cache-file-for-range-read=true", "--file-cache-enable-parallel-downloads=false", "--file-cache-enable-o-direct=false", fmt.Sprintf("--cache-dir=%s/gcsfuse-tmp/TestCacheFileForExcludeRegexTest", setup.TestDir()), fmt.Sprintf("--log-file=%s/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log", setup.TestDir()), "--log-severity=TRACE"},
103+
[]string{fmt.Sprintf("--file-cache-exclude-regex=^%s/%s/", setup.TestBucket(), onlyDirMounted), "--file-cache-max-size-mb=50", "--file-cache-cache-file-for-range-read=true", "--file-cache-enable-parallel-downloads=false", "--file-cache-enable-o-direct=false", fmt.Sprintf("--cache-dir=%s/gcsfuse-tmp/TestCacheFileForExcludeRegexTest", setup.TestDir()), fmt.Sprintf("--log-file=%s/gcsfuse-tmp/TestCacheFileForExcludeRegexTest.log", setup.TestDir()), "--log-severity=TRACE", "--client-protocol=grpc"},
104104
)
105105
}
106106
for _, ts.flags = range flagsSet {

‎tools/integration_tests/read_cache/cache_file_for_include_regex_test.go‎

Lines changed: 136 additions & 185 deletions
Original file line numberDiff line numberDiff line change
@@ -1,188 +1,139 @@
1-
// // Copyright 2025 Google LLC
2-
// //
3-
// // Licensed under the Apache License, Version 2.0 (the "License");
4-
// // you may not use this file except in compliance with the License.
5-
// // You may obtain a copy of the License at
6-
// //
7-
// // http://www.apache.org/licenses/LICENSE-2.0
8-
// //
9-
// // Unless required by applicable law or agreed to in writing, software
10-
// // distributed under the License is distributed on an "AS IS" BASIS,
11-
// // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12-
// // See the License for the specific language governing permissions and
13-
// // limitations under the License.
14-
package read_cache
15-
16-
//
17-
//import (
18-
// "context"
19-
// "fmt"
20-
// "log"
21-
// "path"
22-
// "strings"
23-
// "testing"
24-
//
25-
// "cloud.google.com/go/storage"
26-
// "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client"
27-
// "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs"
28-
// "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations"
29-
// "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup"
30-
//)
31-
//
32-
//////////////////////////////////////////////////////////////////////////
33-
//// Boilerplate
34-
//////////////////////////////////////////////////////////////////////////
35-
//
36-
//type cacheFileForIncludeRegexTest struct {
37-
// flags []string
38-
// storageClient *storage.Client
39-
// ctx context.Context
40-
//}
41-
//
42-
//////////////////////////////////////////////////////////////////////////
43-
//// Test scenarios
44-
//////////////////////////////////////////////////////////////////////////
45-
//
46-
//func TestCacheFileForIncludeRegex_ForIncludedFile(t *testing.T) {
47-
// s := setupTest(t, setup.TestBucket(), "")
48-
// testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSizeForRangeRead, t)
49-
//
50-
// // Read the file and validate that it is cached.
51-
// expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, zeroOffset, t)
52-
// expectedOutcome2 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset1000, t)
53-
//
54-
// structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t)
55-
// validate(expectedOutcome1, structuredReadLogs[0], true, false, 1, t)
56-
// validate(expectedOutcome2, structuredReadLogs[1], false, true, 1, t)
57-
// validateFileIsCached(testFileName, t)
58-
//}
59-
//
60-
//func TestCacheFileForIncludeRegex_ForNonIncludedFile(t *testing.T) {
61-
// s := setupTest(t, "non-matching-regex", "")
62-
// testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSizeForRangeRead, t)
63-
//
64-
// // Read the file and validate that it is not cached.
65-
// expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, zeroOffset, t)
66-
// expectedOutcome2 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset1000, t)
67-
//
68-
// structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t)
69-
// validate(expectedOutcome1, structuredReadLogs[0], true, false, 1, t)
70-
// validate(expectedOutcome2, structuredReadLogs[1], false, false, 1, t)
71-
// validateFileIsNotCached(testFileName, t)
72-
//}
73-
//
74-
//func TestCacheFileForIncludeRegex_ForIncludedAndExcludeOverlap(t *testing.T) {
75-
// //Prioirty is given to exclude and then include when both is defined
76-
// s := setupTest(t, setup.TestBucket(), setup.TestBucket())
77-
// testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSizeForRangeRead, t)
78-
// testFileNameExclude := setupFileInTestDir(s.ctx, s.storageClient, fileSizeForRangeRead, t)
79-
//
80-
// // Read the file and validate that it is cached.
81-
// expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, zeroOffset, t)
82-
// expectedOutcome2 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset1000, t)
83-
//
84-
// structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t)
85-
// validate(expectedOutcome1, structuredReadLogs[0], true, false, 1, t)
86-
// //Include will not hit in cache due to overlapping regex matching with exclude
87-
// validate(expectedOutcome2, structuredReadLogs[1], false, false, 1, t)
88-
// validateFileIsNotCached(testFileName, t)
89-
//
90-
// expectedOutcome1Exclude := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileNameExclude, zeroOffset, t)
91-
// expectedOutcome2Exclude := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileNameExclude, offset1000, t)
92-
//
93-
// structuredReadLogsExclude := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t)
94-
// validate(expectedOutcome1Exclude, structuredReadLogsExclude[2], true, false, 1, t)
95-
// validate(expectedOutcome2Exclude, structuredReadLogsExclude[3], false, false, 1, t)
96-
// validateFileIsNotCached(testFileNameExclude, t)
97-
//
98-
//}
99-
//
100-
//func TestCacheFileForIncludeRegex_ForIncludedAndExcludeNoOverlap(t *testing.T) {
101-
// testExcludeFileName := "bar"
102-
// s := setupTest(t, setup.TestBucket(), setup.TestBucket()+"/.*/"+testExcludeFileName+".*")
103-
// testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSizeForRangeRead, t)
104-
// testFileNameExclude := setupExcludeFileInTestDir(s.ctx, s.storageClient, fileSizeForRangeRead, t)
105-
//
106-
// // Read the file and validate that it is cached.
107-
// expectedOutcome1 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, zeroOffset, t)
108-
// expectedOutcome2 := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileName, offset1000, t)
109-
//
110-
// structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t)
111-
// validate(expectedOutcome1, structuredReadLogs[0], true, false, 1, t)
112-
// validate(expectedOutcome2, structuredReadLogs[1], false, true, 1, t)
113-
// validateFileIsCached(testFileName, t)
114-
//
115-
// expectedOutcome1Exclude := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileNameExclude, zeroOffset, t)
116-
// expectedOutcome2Exclude := readChunkAndValidateObjectContentsFromGCS(s.ctx, s.storageClient, testFileNameExclude, offset1000, t)
117-
//
118-
// structuredReadLogsExclude := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), t)
119-
// validate(expectedOutcome1Exclude, structuredReadLogsExclude[2], true, false, 1, t)
120-
// validate(expectedOutcome2Exclude, structuredReadLogsExclude[3], false, false, 1, t)
121-
// validateFileIsNotCached(testFileNameExclude, t)
122-
//
123-
//}
124-
//
125-
//func setupTest(t *testing.T, includeRegex string, excludeRegex string) *cacheFileForIncludeRegexTest {
126-
//
127-
// ramCacheDir := path.Join("/dev/shm", cacheDirName)
128-
//
129-
// flagTest := gcsfuseTestFlags{
130-
// cliFlags: []string{},
131-
// cacheSize: cacheCapacityForRangeReadTestInMiB,
132-
// cacheFileForRangeRead: true,
133-
// fileName: configFileName,
134-
// cacheDirPath: ramCacheDir,
135-
// }
136-
// if includeRegex != "" {
137-
// if strings.ContainsRune(includeRegex, '/') {
138-
// flagTest.cliFlags = append(flagTest.cliFlags, fmt.Sprintf("--file-cache-include-regex=^%s", includeRegex))
139-
// } else {
140-
// flagTest.cliFlags = append(flagTest.cliFlags, fmt.Sprintf("--file-cache-include-regex=^%s/", includeRegex))
141-
// }
142-
// }
143-
// if excludeRegex != "" {
144-
// if strings.ContainsRune(excludeRegex, '/') {
145-
// flagTest.cliFlags = append(flagTest.cliFlags, fmt.Sprintf("--file-cache-exclude-regex=^%s", excludeRegex))
146-
// } else {
147-
// flagTest.cliFlags = append(flagTest.cliFlags, fmt.Sprintf("--file-cache-exclude-regex=^%s/", excludeRegex))
148-
// }
149-
// }
1+
// Copyright 2025 Google LLC
1502
//
151-
// ts := &cacheFileForIncludeRegexTest{
152-
// ctx: context.Background(),
153-
// flags: flagTest.cliFlags,
154-
// }
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
1556
//
156-
// // Create storage client before running tests.
157-
// closeStorageClient := client.CreateStorageClientWithCancel(&ts.ctx, &ts.storageClient)
7+
// http://www.apache.org/licenses/LICENSE-2.0
1588
//
159-
// //Add flag to mount
160-
// flagTest = appendClientProtocolConfigToFlagSet([]gcsfuseTestFlags{flagTest})[0]
161-
// configFilePath := createConfigFile(&flagTest)
162-
// ts.flags = []string{"--config-file=" + configFilePath}
163-
// if flagTest.cliFlags != nil {
164-
// ts.flags = append(ts.flags, flagTest.cliFlags...)
165-
// }
166-
//
167-
// t.Cleanup(func() {
168-
// t.Logf("Tearing down %s", t.Name())
169-
// err := closeStorageClient()
170-
// if err != nil {
171-
// t.Errorf("closeStorageClient failed: %v", err)
172-
// }
173-
//
174-
// setup.SaveGCSFuseLogFileInCaseOfFailure(t)
175-
// setup.UnmountGCSFuseAndDeleteLogFile(rootDir)
176-
// })
177-
//
178-
// //Setup for mounted directory tests
179-
// setupForMountedDirectoryTests()
180-
// //Clean up the cache directory path as gcsfuse don't clean up on mounting.
181-
// operations.RemoveDir(cacheDirPath)
182-
// //Run with cache directory pointing to RAM based dir
183-
// mountGCSFuseAndSetupTestDir(ts.flags, ts.ctx, ts.storageClient)
184-
//
185-
// log.Printf("Running %s with flags: %+v", t.Name(), ts.flags)
186-
//
187-
// return ts
188-
//}
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
package read_cache
15+
16+
import (
17+
"context"
18+
"log"
19+
"os"
20+
"testing"
21+
22+
"cloud.google.com/go/storage"
23+
"github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client"
24+
"github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/log_parser/json_parser/read_logs"
25+
"github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/operations"
26+
"github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup"
27+
"github.com/stretchr/testify/require"
28+
"github.com/stretchr/testify/suite"
29+
)
30+
31+
////////////////////////////////////////////////////////////////////////
32+
// Boilerplate
33+
////////////////////////////////////////////////////////////////////////
34+
35+
type cacheFileForIncludeRegexTest struct {
36+
flags []string
37+
storageClient *storage.Client
38+
ctx context.Context
39+
baseTestName string
40+
suite.Suite
41+
}
42+
43+
func (s *cacheFileForIncludeRegexTest) SetupSuite() {
44+
setupLogFileAndCacheDir(s.baseTestName)
45+
mountGCSFuseAndSetupTestDir(s.flags, s.ctx, s.storageClient)
46+
}
47+
48+
func (s *cacheFileForIncludeRegexTest) SetupTest() {
49+
//Truncate log file created.
50+
err := os.Truncate(testEnv.cfg.LogFile, 0)
51+
require.NoError(s.T(), err)
52+
// Clean up the cache directory path as gcsfuse don't clean up on mounting.
53+
operations.RemoveDir(testEnv.cacheDirPath)
54+
testEnv.testDirPath = client.SetupTestDirectory(s.ctx, s.storageClient, testDirName)
55+
}
56+
57+
func (s *cacheFileForIncludeRegexTest) TearDownTest() {
58+
setup.SaveGCSFuseLogFileInCaseOfFailure(s.T())
59+
}
60+
61+
func (s *cacheFileForIncludeRegexTest) TearDownSuite() {
62+
setup.UnmountGCSFuseWithConfig(testEnv.cfg)
63+
}
64+
65+
////////////////////////////////////////////////////////////////////////
66+
// Test scenarios
67+
////////////////////////////////////////////////////////////////////////
68+
69+
func (s *cacheFileForIncludeRegexTest) TestCacheFileForIncludeRegexForIncludedFile() {
70+
testFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSize, s.T())
71+
72+
// Read the file and validate that it is cached.
73+
expectedOutcome1 := readFileAndGetExpectedOutcome(testEnv.testDirPath, testFileName, true, 0, s.T())
74+
expectedOutcome2 := readFileAndGetExpectedOutcome(testEnv.testDirPath, testFileName, true, 0, s.T())
75+
76+
structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), s.T())
77+
validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, s.T())
78+
validate(expectedOutcome2, structuredReadLogs[1], true, true, chunksRead, s.T())
79+
validateFileIsCached(testFileName, s.T())
80+
}
81+
82+
func (s *cacheFileForIncludeRegexTest) TestCacheFileForIncludeRegexForNonIncludedFile() {
83+
testFileName := "non-matching-regex" + setup.GenerateRandomString(testFileNameSuffixLength)
84+
client.SetupFileInTestDirectory(s.ctx, s.storageClient, testDirName, testFileName, fileSize, s.T())
85+
86+
// Read the file and validate that it is not cached.
87+
expectedOutcome1 := readFileAndGetExpectedOutcome(testEnv.testDirPath, testFileName, true, 0, s.T())
88+
expectedOutcome2 := readFileAndGetExpectedOutcome(testEnv.testDirPath, testFileName, true, 0, s.T())
89+
90+
structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), s.T())
91+
validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, s.T())
92+
validate(expectedOutcome2, structuredReadLogs[1], true, false, chunksRead, s.T())
93+
validateFileIsNotCached(testFileName, s.T())
94+
}
95+
96+
func (s *cacheFileForIncludeRegexTest) TestCacheFileForIncludeRegexForIncludedAndExcludeNoOverlap() {
97+
includedFileName := setupFileInTestDir(s.ctx, s.storageClient, fileSize, s.T())
98+
excludedFileName := "non-matching-regex" + setup.GenerateRandomString(testFileNameSuffixLength)
99+
client.SetupFileInTestDirectory(s.ctx, s.storageClient, testDirName, excludedFileName, fileSize, s.T())
100+
101+
// Read the included file and validate that it is cached.
102+
expectedOutcome1 := readFileAndGetExpectedOutcome(testEnv.testDirPath, includedFileName, true, 0, s.T())
103+
expectedOutcome2 := readFileAndGetExpectedOutcome(testEnv.testDirPath, includedFileName, true, 0, s.T())
104+
// Read the excluded file and validate that it is not cached.
105+
expectedOutcome3 := readFileAndGetExpectedOutcome(testEnv.testDirPath, excludedFileName, true, 0, s.T())
106+
expectedOutcome4 := readFileAndGetExpectedOutcome(testEnv.testDirPath, excludedFileName, true, 0, s.T())
107+
108+
structuredReadLogs := read_logs.GetStructuredLogsSortedByTimestamp(setup.LogFile(), s.T())
109+
validate(expectedOutcome1, structuredReadLogs[0], true, false, chunksRead, s.T())
110+
validate(expectedOutcome2, structuredReadLogs[1], true, true, chunksRead, s.T())
111+
validateFileIsCached(includedFileName, s.T())
112+
validate(expectedOutcome3, structuredReadLogs[2], true, false, chunksRead, s.T())
113+
validate(expectedOutcome4, structuredReadLogs[3], true, false, chunksRead, s.T())
114+
validateFileIsNotCached(excludedFileName, s.T())
115+
}
116+
117+
////////////////////////////////////////////////////////////////////////
118+
// Test Function (Runs once before all tests)
119+
////////////////////////////////////////////////////////////////////////
120+
121+
func TestCacheFileForIncludeRegexTest(t *testing.T) {
122+
ts := &cacheFileForIncludeRegexTest{
123+
ctx: context.Background(),
124+
storageClient: testEnv.storageClient,
125+
baseTestName: t.Name(),
126+
}
127+
// Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config.
128+
if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" {
129+
suite.Run(t, ts)
130+
return
131+
}
132+
133+
// Run tests for GCE environment otherwise.
134+
flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name())
135+
for _, ts.flags = range flagsSet {
136+
log.Printf("Running tests with flags: %s", ts.flags)
137+
suite.Run(t, ts)
138+
}
139+
}

‎tools/integration_tests/read_cache/helpers_test.go‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -259,13 +259,6 @@ func setupFileInTestDir(ctx context.Context, storageClient *storage.Client, file
259259
return testFileName
260260
}
261261

262-
func setupExcludeFileInTestDir(ctx context.Context, storageClient *storage.Client, fileSize int64, t *testing.T) (fileName string) {
263-
testFileName := testExcludeFileName + setup.GenerateRandomString(testFileNameSuffixLength)
264-
client.SetupFileInTestDirectory(ctx, storageClient, testDirName, testFileName, fileSize, t)
265-
266-
return testFileName
267-
}
268-
269262
func runTestsOnlyForDynamicMount(t *testing.T) {
270263
if !strings.Contains(setup.MntDir(), setup.TestBucket()) {
271264
log.Println("This test will run only for dynamic mounting...")

0 commit comments

Comments
 (0)