diff --git a/cmd/fs/ls.go b/cmd/fs/ls.go index ac19238573..e53a6f43e4 100644 --- a/cmd/fs/ls.go +++ b/cmd/fs/ls.go @@ -1,23 +1,76 @@ package fs import ( - "fmt" + "path" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/filer" "github.com/spf13/cobra" ) +func expandPath(i filer.FileInfo, root string) filer.FileInfo { + i.Name = path.Join(root, i.Name) + return i +} + // lsCmd represents the ls command var lsCmd = &cobra.Command{ - Use: "ls ", - Short: "Lists files", - Long: `Lists files`, - Hidden: true, + Use: "ls ", + Short: "Lists files", + Long: `Lists files in a DBFS or WSFS directory`, + Args: cobra.MaximumNArgs(1), + Annotations: map[string]string{}, + PreRunE: root.MustWorkspaceClient, RunE: func(cmd *cobra.Command, args []string) error { - return fmt.Errorf("TODO") + // Assign template according to whether -l is specified + template := cmdio.Heredoc(` + {{range .}}{{.Name}} + {{end}} + `) + if longMode { + template = cmdio.Heredoc(` + {{range .}}{{.Type|printf "%-10s"}} {{.Size}} {{.ModTime|unix_date}} {{.Name}} + {{end}} + `) + } + + // Path to list files from. Defaults to`/` + rootPath := "/" + if len(args) > 0 { + rootPath = args[0] + } + + // Initialize workspace client + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + f, err := filer.NewWorkspaceFilesClient(w, rootPath) + if err != nil { + return err + } + + // Get file info + filesInfo, err := f.ReadDir(ctx, "") + if err != nil { + return err + } + + // compute output with expanded paths if necessary + if absolute { + for i := range filesInfo { + filesInfo[i] = expandPath(filesInfo[i], rootPath) + } + } + return cmdio.RenderWithTemplate(ctx, filesInfo, template) }, } +var longMode bool +var absolute bool + func init() { + lsCmd.Flags().BoolVarP(&longMode, "long", "l", false, "Displays full information including size, file type and modification time since Epoch in milliseconds.") + lsCmd.Flags().BoolVar(&absolute, "absolute", false, "Displays absolute paths.") fsCmd.AddCommand(lsCmd) } diff --git a/internal/fs/ls_test.go b/internal/fs/ls_test.go new file mode 100644 index 0000000000..757fa72355 --- /dev/null +++ b/internal/fs/ls_test.go @@ -0,0 +1,85 @@ +package fs + +import ( + "encoding/json" + "fmt" + "path" + "testing" + + _ "github.com/databricks/cli/cmd/fs" + "github.com/databricks/cli/internal" + "github.com/databricks/cli/internal/helpers" + "github.com/databricks/cli/libs/cmdio" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func assertObjectListed(t *testing.T, parsedLogs []map[string]string, name string, objectType string) { + foundFile := false + for _, v := range parsedLogs { + if v["Name"] != name { + continue + } + foundFile = true + assert.Equal(t, objectType, v["Type"]) + } + assert.True(t, foundFile, fmt.Sprintf("failed to find file %s in output logs", name)) +} + +func TestAccFsLs(t *testing.T) { + t.Log(internal.GetEnvOrSkipTest(t, "CLOUD_ENV")) + + // setup some testdata in the workspace + w := helpers.NewWorkspaceTestdata(t) + w.AddFile("foo.txt", `hello, world`) + w.AddFile("python_notebook.py", cmdio.Heredoc(` + #Databricks notebook source + print(2)`)) + w.AddFile("python_file.py", `print(1)`) + w.Mkdir("my_directory") + w.AddFile("my_directory/.gitkeep", "") + + // run list command + stdout, stderr := internal.RequireSuccessfulRun(t, "fs", "ls", w.RootPath(), "--output=json") + + // read and parse the output logs + parsedLogs := make([]map[string]string, 0) + err := json.Unmarshal(stdout.Bytes(), &parsedLogs) + require.NoError(t, err) + + // make assertions on the output logs + assert.Equal(t, stderr.String(), "") + assertObjectListed(t, parsedLogs, "python_file.py", "FILE") + assertObjectListed(t, parsedLogs, "foo.txt", "FILE") + assertObjectListed(t, parsedLogs, "python_notebook", "NOTEBOOK") + assertObjectListed(t, parsedLogs, "my_directory", "DIRECTORY") +} + +func TestAccFsLsWithAbsoluteFlag(t *testing.T) { + t.Log(internal.GetEnvOrSkipTest(t, "CLOUD_ENV")) + + // setup some testdata in the workspace + w := helpers.NewWorkspaceTestdata(t) + w.AddFile("foo.txt", `hello, world`) + w.AddFile("python_notebook.py", cmdio.Heredoc(` + #Databricks notebook source + print(2)`)) + w.AddFile("python_file.py", `print(1)`) + w.Mkdir("my_directory") + w.AddFile("my_directory/.gitkeep", "") + + // run list command + stdout, stderr := internal.RequireSuccessfulRun(t, "fs", "ls", w.RootPath(), "--output=json", "--absolute") + + // read and parse the output logs + parsedLogs := make([]map[string]string, 0) + err := json.Unmarshal(stdout.Bytes(), &parsedLogs) + require.NoError(t, err) + + // make assertions on the output logs + assert.Equal(t, stderr.String(), "") + assertObjectListed(t, parsedLogs, path.Join(w.RootPath(), "python_file.py"), "FILE") + assertObjectListed(t, parsedLogs, path.Join(w.RootPath(), "foo.txt"), "FILE") + assertObjectListed(t, parsedLogs, path.Join(w.RootPath(), "python_notebook"), "NOTEBOOK") + assertObjectListed(t, parsedLogs, path.Join(w.RootPath(), "my_directory"), "DIRECTORY") +} diff --git a/internal/helpers/workspace_testdata.go b/internal/helpers/workspace_testdata.go new file mode 100644 index 0000000000..8548037801 --- /dev/null +++ b/internal/helpers/workspace_testdata.go @@ -0,0 +1,88 @@ +package helpers + +import ( + "context" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "testing" + + "github.com/databricks/cli/internal" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/stretchr/testify/require" +) + +type workspaceTestdata struct { + root string + t *testing.T + client *databricks.WorkspaceClient +} + +func NewWorkspaceTestdata(t *testing.T) *workspaceTestdata { + ctx := context.Background() + w := databricks.Must(databricks.NewWorkspaceClient()) + + me, err := w.CurrentUser.Me(ctx) + require.NoError(t, err) + path := fmt.Sprintf("/Users/%s/%s", me.UserName, internal.RandomName("wsfs-files-")) + + // Ensure directory exists, but doesn't exist YET! + // Otherwise we could inadvertently remove a directory that already exists on cleanup. + t.Logf("mkdir %s", path) + err = w.Workspace.MkdirsByPath(ctx, path) + require.NoError(t, err) + + // Remove test directory on test completion. + t.Cleanup(func() { + t.Logf("rm -rf %s", path) + err := w.Workspace.Delete(ctx, workspace.Delete{ + Path: path, + Recursive: true, + }) + if err == nil || apierr.IsMissing(err) { + return + } + t.Logf("unable to remove temporary workspace path %s: %#v", path, err) + }) + + return &workspaceTestdata{ + root: path, + t: t, + client: w, + } +} + +func (w *workspaceTestdata) RootPath() string { + return w.root +} + +func (w *workspaceTestdata) AddFile(name string, content string) { + path := path.Join(w.root, name) + ctx := context.Background() + + // url path for uploading file API + urlPath := fmt.Sprintf( + "/api/2.0/workspace-files/import-file/%s?overwrite=true", + url.PathEscape(strings.TrimLeft(path, "/")), + ) + + // initialize API client + apiClient, err := client.New(w.client.Config) + require.NoError(w.t, err) + + // Make API request + err = apiClient.Do(ctx, http.MethodPost, urlPath, content, nil) + require.NoError(w.t, err) +} + +func (w *workspaceTestdata) Mkdir(name string) { + ctx := context.Background() + path := path.Join(w.root, name) + err := w.client.Workspace.MkdirsByPath(ctx, path) + require.NoError(w.t, err) +} diff --git a/libs/cmdio/io.go b/libs/cmdio/io.go index e5a7199037..8326557fcb 100644 --- a/libs/cmdio/io.go +++ b/libs/cmdio/io.go @@ -71,6 +71,18 @@ func (c *cmdIO) Render(v any) error { } } +func RenderWithTemplate(ctx context.Context, v any, template string) error { + c := fromContext(ctx) + switch c.outputFormat { + case flags.OutputJSON: + return renderJson(c.out, v) + case flags.OutputText: + return renderTemplate(c.out, template, v) + default: + return fmt.Errorf("invalid output format: %s", c.outputFormat) + } +} + func Render(ctx context.Context, v any) error { c := fromContext(ctx) return c.Render(v) diff --git a/libs/cmdio/render.go b/libs/cmdio/render.go index 8aff2b8d2c..70e26d7160 100644 --- a/libs/cmdio/render.go +++ b/libs/cmdio/render.go @@ -6,6 +6,7 @@ import ( "strings" "text/tabwriter" "text/template" + "time" "github.com/fatih/color" "github.com/nwidger/jsoncolor" @@ -86,6 +87,9 @@ func renderTemplate(w io.Writer, tmpl string, v any) error { } return string(b), nil }, + "unix_date": func(t time.Time) string { + return t.UTC().Format(time.UnixDate) + }, }).Parse(tmpl) if err != nil { return err diff --git a/libs/filer/filer.go b/libs/filer/filer.go index 92de6e128c..a29970ac75 100644 --- a/libs/filer/filer.go +++ b/libs/filer/filer.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "time" ) type WriteMode int @@ -13,6 +14,22 @@ const ( CreateParentDirectories = iota << 1 ) +// This struct is an abstract over file information from different file +// systems like WSFS and DBFS. The names for the fields are inspired from https://pkg.go.dev/io/fs#FileInfo +type FileInfo struct { + // The type of the file in workspace + Type string + + // Base name of the file + Name string + + // Size in bytes of the file + Size int64 + + // Last Modified time of the file + ModTime time.Time +} + type FileAlreadyExistsError struct { path string } @@ -41,4 +58,10 @@ type Filer interface { // Delete file at `path`. Delete(ctx context.Context, path string) error + + // Return contents of directory at `path` + ReadDir(ctx context.Context, path string) ([]FileInfo, error) + + // Creates directory at `path`, creating any intermediate directories as required + Mkdir(ctx context.Context, path string) error } diff --git a/libs/filer/root_path.go b/libs/filer/root_path.go index 65b26d5310..bdeff5d73c 100644 --- a/libs/filer/root_path.go +++ b/libs/filer/root_path.go @@ -30,10 +30,5 @@ func (p *RootPath) Join(name string) (string, error) { return "", fmt.Errorf("relative path escapes root: %s", name) } - // Don't allow name to resolve to the root path. - if strings.TrimPrefix(absPath, p.rootPath) == "" { - return "", fmt.Errorf("relative path resolves to root: %s", name) - } - return absPath, nil } diff --git a/libs/filer/root_path_test.go b/libs/filer/root_path_test.go index 3787ef36bf..965842d030 100644 --- a/libs/filer/root_path_test.go +++ b/libs/filer/root_path_test.go @@ -31,6 +31,26 @@ func testRootPath(t *testing.T, uncleanRoot string) { assert.NoError(t, err) assert.Equal(t, cleanRoot+"/a/b/f/g", remotePath) + remotePath, err = rp.Join(".//a/..//./b/..") + assert.NoError(t, err) + assert.Equal(t, cleanRoot, remotePath) + + remotePath, err = rp.Join("a/b/../..") + assert.NoError(t, err) + assert.Equal(t, cleanRoot, remotePath) + + remotePath, err = rp.Join("") + assert.NoError(t, err) + assert.Equal(t, cleanRoot, remotePath) + + remotePath, err = rp.Join(".") + assert.NoError(t, err) + assert.Equal(t, cleanRoot, remotePath) + + remotePath, err = rp.Join("/") + assert.NoError(t, err) + assert.Equal(t, cleanRoot, remotePath) + _, err = rp.Join("..") assert.ErrorContains(t, err, `relative path escapes root: ..`) @@ -57,21 +77,6 @@ func testRootPath(t *testing.T, uncleanRoot string) { _, err = rp.Join("../..") assert.ErrorContains(t, err, `relative path escapes root: ../..`) - - _, err = rp.Join(".//a/..//./b/..") - assert.ErrorContains(t, err, `relative path resolves to root: .//a/..//./b/..`) - - _, err = rp.Join("a/b/../..") - assert.ErrorContains(t, err, "relative path resolves to root: a/b/../..") - - _, err = rp.Join("") - assert.ErrorContains(t, err, "relative path resolves to root: ") - - _, err = rp.Join(".") - assert.ErrorContains(t, err, "relative path resolves to root: .") - - _, err = rp.Join("/") - assert.ErrorContains(t, err, "relative path resolves to root: /") } func TestRootPathClean(t *testing.T) { diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index ff813f0913..a6f0c56ef8 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -10,6 +10,7 @@ import ( "net/url" "path" "strings" + "time" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -128,3 +129,38 @@ func (w *WorkspaceFilesClient) Delete(ctx context.Context, name string) error { Recursive: false, }) } + +func (w *WorkspaceFilesClient) ReadDir(ctx context.Context, name string) ([]FileInfo, error) { + absPath, err := w.root.Join(name) + if err != nil { + return nil, err + } + + objects, err := w.workspaceClient.Workspace.ListAll(ctx, workspace.ListWorkspaceRequest{ + Path: absPath, + }) + if err != nil { + return nil, err + } + + info := make([]FileInfo, len(objects)) + for i, v := range objects { + info[i] = FileInfo{ + Type: string(v.ObjectType), + Name: path.Base(v.Path), + Size: v.Size, + ModTime: time.UnixMilli(v.ModifiedAt), + } + } + return info, nil +} + +func (w *WorkspaceFilesClient) Mkdir(ctx context.Context, name string) error { + dirPath, err := w.root.Join(name) + if err != nil { + return err + } + return w.workspaceClient.Workspace.Mkdirs(ctx, workspace.Mkdirs{ + Path: dirPath, + }) +}