Skip to content

Commit f057846

Browse files
KirCutexrgzs
KirCute
and
xrgzs
authored
feat(drivers): add autoindex driver (#1978)
* feat(drivers): add autoindex driver * fix * add NoUpload Signed-off-by: MadDogOwner <xiaoran@xrgzs.top> * add TestParseSize Signed-off-by: MadDogOwner <xiaoran@xrgzs.top> * go mod tidy Signed-off-by: MadDogOwner <xiaoran@xrgzs.top> * use base.RestyClient Signed-off-by: MadDogOwner <xiaoran@xrgzs.top> * fix: support evaluate size and modified time * fix apache * perf * feat: support ignore size and modified time * rename driver --------- Signed-off-by: MadDogOwner <xiaoran@xrgzs.top> Co-authored-by: MadDogOwner <xiaoran@xrgzs.top>
1 parent 27fdd03 commit f057846

File tree

8 files changed

+386
-2
lines changed

8 files changed

+386
-2
lines changed

‎drivers/all.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
_ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive"
1818
_ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_open"
1919
_ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_share"
20+
_ "github.com/OpenListTeam/OpenList/v4/drivers/autoindex"
2021
_ "github.com/OpenListTeam/OpenList/v4/drivers/azure_blob"
2122
_ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_netdisk"
2223
_ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_photo"

‎drivers/autoindex/driver.go‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
package autoindex
2+
3+
import (
4+
"context"
5+
"strings"
6+
"time"
7+
8+
"github.com/OpenListTeam/OpenList/v4/drivers/base"
9+
"github.com/OpenListTeam/OpenList/v4/internal/driver"
10+
"github.com/OpenListTeam/OpenList/v4/internal/model"
11+
"github.com/OpenListTeam/OpenList/v4/internal/op"
12+
"github.com/antchfx/htmlquery"
13+
"github.com/antchfx/xpath"
14+
"github.com/pkg/errors"
15+
log "github.com/sirupsen/logrus"
16+
)
17+
18+
type AutoIndex struct {
19+
model.Storage
20+
Addition
21+
itemXPath *xpath.Expr
22+
nameXPath *xpath.Expr
23+
modifiedXPath *xpath.Expr
24+
sizeXPath *xpath.Expr
25+
ignores map[string]any
26+
}
27+
28+
func (d *AutoIndex) Config() driver.Config {
29+
return config
30+
}
31+
32+
func (d *AutoIndex) GetAddition() driver.Additional {
33+
return &d.Addition
34+
}
35+
36+
func (d *AutoIndex) Init(ctx context.Context) error {
37+
var err error
38+
d.itemXPath, err = xpath.Compile(d.ItemXPath)
39+
if err != nil {
40+
return errors.WithMessage(err, "failed to compile Item XPath")
41+
}
42+
d.nameXPath, err = xpath.Compile(d.NameXPath)
43+
if err != nil {
44+
return errors.WithMessage(err, "failed to compile Name XPath")
45+
}
46+
if len(d.ModifiedXPath) > 0 {
47+
d.modifiedXPath, err = xpath.Compile(d.ModifiedXPath)
48+
if err != nil {
49+
return errors.WithMessage(err, "failed to compile Modified XPath")
50+
}
51+
}
52+
if len(d.SizeXPath) > 0 {
53+
d.sizeXPath, err = xpath.Compile(d.SizeXPath)
54+
if err != nil {
55+
return errors.WithMessage(err, "failed to compile Size XPath")
56+
}
57+
}
58+
ignores := strings.Split(d.IgnoreFileNames, "\n")
59+
d.ignores = make(map[string]any, len(ignores))
60+
for _, i := range ignores {
61+
i = strings.TrimSpace(i)
62+
if len(i) == 0 {
63+
continue
64+
}
65+
d.ignores[i] = struct{}{}
66+
}
67+
hasScheme := strings.Contains(d.URL, "://")
68+
hasSuffix := strings.HasSuffix(d.URL, "/")
69+
if !hasScheme || !hasSuffix {
70+
if !hasSuffix {
71+
d.URL = d.URL + "/"
72+
}
73+
if !hasScheme {
74+
d.URL = "https://" + d.URL
75+
}
76+
op.MustSaveDriverStorage(d)
77+
}
78+
return nil
79+
}
80+
81+
func (d *AutoIndex) Drop(ctx context.Context) error {
82+
return nil
83+
}
84+
85+
func (d *AutoIndex) GetRoot(ctx context.Context) (model.Obj, error) {
86+
return &model.Object{
87+
Name: op.RootName,
88+
Path: d.URL,
89+
Modified: d.Modified,
90+
Mask: model.Locked,
91+
IsFolder: true,
92+
}, nil
93+
}
94+
95+
func (d *AutoIndex) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
96+
res, err := base.RestyClient.R().
97+
SetContext(ctx).
98+
SetDoNotParseResponse(true).
99+
Get(dir.GetPath())
100+
if err != nil {
101+
return nil, errors.WithMessagef(err, "failed to get url [%s]", dir.GetPath())
102+
}
103+
defer res.RawResponse.Body.Close()
104+
doc, err := htmlquery.Parse(res.RawBody())
105+
if err != nil {
106+
return nil, errors.WithMessagef(err, "failed to parse [%s]", dir.GetPath())
107+
}
108+
itemsIter := d.itemXPath.Select(htmlquery.CreateXPathNavigator(doc))
109+
var objs []model.Obj
110+
for itemsIter.MoveNext() {
111+
nameFull, err := parseString(d.nameXPath.Evaluate(itemsIter.Current().Copy()))
112+
if err != nil {
113+
log.Warnf("skip invalid name evaluating result: %v", err)
114+
continue
115+
}
116+
nameFull = strings.TrimSpace(nameFull)
117+
name, isDir := strings.CutSuffix(nameFull, "/")
118+
if _, ok := d.ignores[name]; ok {
119+
continue
120+
}
121+
var size int64 = 0
122+
exact := false
123+
modified := time.Now()
124+
if d.sizeXPath != nil {
125+
size, exact, err = parseSize(d.sizeXPath.Evaluate(itemsIter.Current().Copy()))
126+
if err != nil {
127+
log.Errorf("failed to parse size of %s: %v", name, err)
128+
}
129+
}
130+
if d.modifiedXPath != nil {
131+
modified, err = parseTime(d.modifiedXPath.Evaluate(itemsIter.Current().Copy()), d.ModifiedTimeFormat)
132+
if err != nil {
133+
log.Errorf("failed to parse modified time of %s: %v", name, err)
134+
}
135+
}
136+
var o model.Obj = &model.Object{
137+
Name: name,
138+
IsFolder: isDir,
139+
Path: dir.GetPath() + nameFull,
140+
Modified: modified,
141+
Size: size,
142+
}
143+
if exact {
144+
o = &exactSizeObj{Obj: o}
145+
}
146+
objs = append(objs, o)
147+
}
148+
return objs, nil
149+
}
150+
151+
func (d *AutoIndex) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
152+
if _, ok := file.(*exactSizeObj); ok || args.Redirect {
153+
return &model.Link{URL: file.GetPath()}, nil
154+
}
155+
res, err := base.RestyClient.R().
156+
SetContext(ctx).
157+
SetDoNotParseResponse(true).
158+
Head(file.GetPath())
159+
if err != nil {
160+
return nil, errors.WithMessagef(err, "failed to head [%s]", file.GetPath())
161+
}
162+
_ = res.RawResponse.Body.Close()
163+
return &model.Link{
164+
URL: file.GetPath(),
165+
ContentLength: res.RawResponse.ContentLength,
166+
}, nil
167+
}
168+
169+
var _ driver.Driver = (*AutoIndex)(nil)

‎drivers/autoindex/meta.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package autoindex
2+
3+
import (
4+
"github.com/OpenListTeam/OpenList/v4/internal/driver"
5+
"github.com/OpenListTeam/OpenList/v4/internal/op"
6+
)
7+
8+
type Addition struct {
9+
URL string `json:"url" required:"true"`
10+
ItemXPath string `json:"item_xpath" required:"true"`
11+
NameXPath string `json:"name_xpath" required:"true"`
12+
ModifiedXPath string `json:"modified_xpath"`
13+
SizeXPath string `json:"size_xpath"`
14+
IgnoreFileNames string `json:"ignore_file_names" type:"text" default:".\n..\nParent Directory\nUp"`
15+
ModifiedTimeFormat string `json:"modified_time_format" default:"02-Jan-2006 15:04" help:"Must be based on the time point Mon Jan 2 15:04:05 -0700 MST 2006"`
16+
}
17+
18+
var config = driver.Config{
19+
Name: "AutoIndex",
20+
LocalSort: true,
21+
CheckStatus: true,
22+
NoUpload: true,
23+
}
24+
25+
func init() {
26+
op.RegisterDriver(func() driver.Driver {
27+
return &AutoIndex{}
28+
})
29+
}

‎drivers/autoindex/types.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package autoindex
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/OpenListTeam/OpenList/v4/internal/model"
7+
)
8+
9+
var (
10+
errEmptyEvaluateResult = fmt.Errorf("empty result")
11+
)
12+
13+
type exactSizeObj struct{ model.Obj }

‎drivers/autoindex/util.go‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package autoindex
2+
3+
import (
4+
"fmt"
5+
"strconv"
6+
"strings"
7+
"time"
8+
9+
"github.com/antchfx/xpath"
10+
"github.com/pkg/errors"
11+
)
12+
13+
var units = map[string]int64{
14+
"": 1,
15+
"b": 1,
16+
"byte": 1,
17+
"bytes": 1,
18+
"k": 1 << 10,
19+
"kb": 1 << 10,
20+
"kib": 1 << 10,
21+
"m": 1 << 20,
22+
"mb": 1 << 20,
23+
"mib": 1 << 20,
24+
"g": 1 << 30,
25+
"gb": 1 << 30,
26+
"gib": 1 << 30,
27+
"t": 1 << 40,
28+
"tb": 1 << 40,
29+
"tib": 1 << 40,
30+
"p": 1 << 50,
31+
"pb": 1 << 50,
32+
"pib": 1 << 50,
33+
}
34+
35+
func splitUnit(s string) (string, string) {
36+
for i := len(s) - 1; i >= 0; i-- {
37+
if s[i] >= '0' && s[i] <= '9' {
38+
return strings.TrimSpace(s[:i+1]), strings.TrimSpace(s[i+1:])
39+
}
40+
}
41+
return "", s
42+
}
43+
44+
func parseSize(a any) (int64, bool, error) {
45+
// 第二个返回值exact表示大小是否精确
46+
if f, ok := a.(float64); ok {
47+
return int64(f), false, nil
48+
}
49+
s, err := parseString(a)
50+
if errors.Is(err, errEmptyEvaluateResult) {
51+
// 可能是错误，也可能确实大小为0
52+
// 如果确实大小为0，大概率不会下载，exact返回false也不会有什么性能损失
53+
// 如果是错误，exact返回true会导致本地代理出错，综合来看返回false更好
54+
return 0, false, nil
55+
}
56+
if err != nil {
57+
return 0, false, err
58+
}
59+
s = strings.TrimSpace(s)
60+
if s == "-" {
61+
return 0, false, nil
62+
}
63+
nbs, unit := splitUnit(s)
64+
mul, ok := units[strings.ToLower(unit)]
65+
exact := mul == 1
66+
if !ok {
67+
mul = 1
68+
// 推测无单位，exact应为false
69+
}
70+
nb, err := strconv.ParseInt(nbs, 10, 64)
71+
if err != nil {
72+
fnb, err := strconv.ParseFloat(nbs, 64)
73+
if err != nil {
74+
return 0, false, fmt.Errorf("failed to convert %s to number", nbs)
75+
}
76+
nb = int64(fnb * float64(mul))
77+
exact = false
78+
} else {
79+
nb = nb * mul
80+
}
81+
return nb, exact, nil
82+
}
83+
84+
func parseString(res any) (string, error) {
85+
if r, ok := res.(string); ok {
86+
if len(r) == 0 {
87+
return "", errEmptyEvaluateResult
88+
}
89+
return r, nil
90+
}
91+
n, ok := res.(*xpath.NodeIterator)
92+
if !ok {
93+
return "", fmt.Errorf("unsupported evaluating result")
94+
}
95+
if !n.MoveNext() {
96+
return "", fmt.Errorf("no matched nodes")
97+
}
98+
ns := n.Current().Value()
99+
if len(ns) == 0 {
100+
return "", errEmptyEvaluateResult
101+
}
102+
return ns, nil
103+
}
104+
105+
func parseTime(res any, format string) (time.Time, error) {
106+
s, err := parseString(res)
107+
if err != nil {
108+
return time.Now(), err
109+
}
110+
s = strings.TrimSpace(s)
111+
t, err := time.Parse(format, s)
112+
if err != nil {
113+
return time.Now(), errors.WithMessagef(err, "failed to convert %s to time", s)
114+
}
115+
return t, nil
116+
}

‎drivers/autoindex/util_test.go‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package autoindex
2+
3+
import (
4+
"testing"
5+
)
6+
7+
type wantType struct {
8+
v int64
9+
exact bool
10+
error bool
11+
}
12+
13+
func TestParseSize(t *testing.T) {
14+
tests := []struct {
15+
input string
16+
want wantType
17+
}{
18+
{"100", wantType{100, true, false}},
19+
{"1k", wantType{1024, false, false}},
20+
{"1kb", wantType{1024, false, false}},
21+
{"1K", wantType{1024, false, false}}, // case insensitive
22+
{"1.5m", wantType{1572864, false, false}}, // 1.5 * 1024^2
23+
{"500 bytes", wantType{500, true, false}},
24+
{"-", wantType{0, false, false}},
25+
{"", wantType{0, false, false}},
26+
{"abc", wantType{0, false, true}},
27+
{"1.5GB", wantType{1610612736, false, false}}, // 1.5 * 1024^3
28+
{"2t", wantType{2199023255552, false, false}}, // 2 * 1024^4
29+
{"1p", wantType{1125899906842624, false, false}}, // 1 * 1024^5
30+
{"0", wantType{0, true, false}},
31+
{" 100 ", wantType{100, true, false}}, // trimmed
32+
{"100b", wantType{100, true, false}},
33+
{"1gib", wantType{1073741824, false, false}}, // 1024^3
34+
{"1z", wantType{1, false, false}}, // invalid unit, mul=1
35+
{"1.5", wantType{1, false, false}}, // float without unit, truncated
36+
{"2.7k", wantType{2764, false, false}}, // 2.7 * 1024 truncated
37+
{"1.0g", wantType{1073741824, false, false}}, // 1.0 * 1024^3
38+
{"invalid", wantType{0, false, true}},
39+
{"123xyz", wantType{123, false, false}}, // unit not found, mul=1
40+
}
41+
for _, tt := range tests {
42+
t.Run(tt.input, func(t *testing.T) {
43+
got, exact, err := parseSize(tt.input)
44+
if got != tt.want.v || exact != tt.want.exact || (err != nil) != tt.want.error {
45+
t.Errorf("ParseSize(%q) = (%d, %t, %t), want (%d, %t, %t)", tt.input, got, exact, err != nil, tt.want.v, tt.want.exact, tt.want.error)
46+
}
47+
})
48+
}
49+
}

0 commit comments

Comments
 (0)