Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .flocks/plugins/skills/web2cli/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ print(js("window.__apiCapture.config.captureMode"))
### 4. 明确需要捕获的功能/操作

- 要求用户手动操作要捕获的页面动作,例如查询、翻页、筛选、提交表单、点击按钮、导出数据。
- 或请求用户描述需要 hook 的操作,便于你直接去页面代替用户执行
- 或者请求用户描述需要 hook 的操作或功能,便于你直接去页面代替用户执行

需要确认捕获是否开始时:

Expand Down
234 changes: 192 additions & 42 deletions .flocks/plugins/skills/web2cli/scripts/generate-cli.py

Large diffs are not rendered by default.

52 changes: 45 additions & 7 deletions .flocks/plugins/skills/web2cli/scripts/generate-spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,19 @@ def parse_json_text(text: str) -> Any:
return {"raw": text}


def get_request_content_type(request: dict[str, Any]) -> str:
"""Return the normalized request content type."""
direct = request.get("requestContentType")
if direct:
return str(direct).lower()

headers = request.get("requestHeaders", {}) or request.get("request_headers", {})
for key, value in headers.items():
if str(key).lower() == "content-type" and value:
return str(value).lower()
return ""


def infer_type(value: Any) -> str:
"""Return a compact type name for spec/verify output."""
if value is None:
Expand Down Expand Up @@ -237,8 +250,10 @@ def collect_columns(item: Any) -> list[dict[str, Any]]:
return columns


def build_templates(request: dict[str, Any], url_info: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any], list[dict[str, Any]]]:
"""Build query/body templates and CLI arg definitions."""
def build_templates(
request: dict[str, Any], url_info: dict[str, Any]
) -> tuple[dict[str, Any], dict[str, Any], list[dict[str, Any]], str, str]:
"""Build query/body templates, payload mode, and CLI arg definitions."""
args: list[dict[str, Any]] = []
seen_args: set[str] = set()

Expand All @@ -264,15 +279,36 @@ def transform_mapping(data: dict[str, Any]) -> dict[str, Any]:
result[key] = value
return result

body = parse_json_text(str(request.get("requestBody", "")))
if not isinstance(body, dict) or "raw" in body:
body = {}
body_text = str(request.get("requestBody", ""))
body_kind = str(request.get("requestBodyKind", "")).lower()
content_type = get_request_content_type(request)
parsed_body = parse_json_text(body_text)
body: dict[str, Any] = {}
payload_mode = "none"
raw_body_template = ""

if isinstance(parsed_body, dict) and "raw" not in parsed_body:
body = parsed_body
if body:
if body_kind in {"urlencoded", "formdata"}:
payload_mode = "form"
elif "application/x-www-form-urlencoded" in content_type or "multipart/form-data" in content_type:
payload_mode = "form"
else:
payload_mode = "json"
elif body_text:
if "application/x-www-form-urlencoded" in content_type:
body = dict(parse_qsl(body_text, keep_blank_values=True))
payload_mode = "form" if body else "raw"
else:
payload_mode = "raw"
raw_body_template = body_text

query_template = transform_mapping(url_info["query"])
body_template = transform_mapping(body)

args.sort(key=lambda item: (0 if item["name"] == "page" else 1 if item["name"] == "limit" else 2, item["name"]))
return query_template, body_template, args
return query_template, body_template, args, payload_mode, raw_body_template


def build_strategy(request: dict[str, Any]) -> tuple[str, dict[str, Any]]:
Expand Down Expand Up @@ -364,7 +400,7 @@ def build_operation_entry(request: dict[str, Any]) -> dict[str, Any]:
response = parse_json_text(str(request.get("response", "")))
collection = find_best_collection(response)
row_item = collection["item"] if collection is not None else response
query_template, body_template, args = build_templates(request, url_info)
query_template, body_template, args, payload_mode, raw_body_template = build_templates(request, url_info)
columns = collect_columns(row_item)

defaults = {item["name"]: item["default"] for item in args}
Expand All @@ -386,6 +422,8 @@ def build_operation_entry(request: dict[str, Any]) -> dict[str, Any]:
"endpoint": pathname,
"queryTemplate": query_template,
"bodyTemplate": body_template,
"payloadMode": payload_mode,
"rawBodyTemplate": raw_body_template,
"headers": safe_headers(request),
"captureSource": request.get("captureSource", "pageHook"),
"captureReason": request.get("captureReason", ""),
Expand Down
129 changes: 110 additions & 19 deletions .flocks/plugins/skills/web2cli/scripts/inject-hook-base.js
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,36 @@
return result;
}

function mergeHeaders(baseHeaders, overrideHeaders) {
var result = {};
var key;
var base = normalizeHeaders(baseHeaders);
var override = normalizeHeaders(overrideHeaders);
for (key in base) {
if (Object.prototype.hasOwnProperty.call(base, key)) {
result[key] = base[key];
}
}
for (key in override) {
if (Object.prototype.hasOwnProperty.call(override, key)) {
result[key] = override[key];
}
}
return result;
}

function getHeader(headers, name) {
var key;
var expected = String(name || '').toLowerCase();
if (!headers) {
return '';
}
return headers[name] || headers[name.toLowerCase()] || '';
for (key in headers) {
if (Object.prototype.hasOwnProperty.call(headers, key) && String(key).toLowerCase() === expected) {
return headers[key];
}
}
return '';
}

function hasStaticExtension(pathname) {
Expand Down Expand Up @@ -194,7 +219,22 @@
};
}

function summarizeBody(body) {
function parseUrlEncodedBody(text) {
var result = {};
try {
if (typeof URLSearchParams !== 'undefined') {
var params = new URLSearchParams(String(text || ''));
params.forEach(function(value, key) {
result[key] = value;
});
}
} catch (error) {
return null;
}
return result;
}

function summarizeBody(body, contentType) {
var result = {
kind: 'empty',
display: '',
Expand Down Expand Up @@ -233,7 +273,17 @@
}

if (typeof body === 'string') {
var normalizedContentType = String(contentType || '').toLowerCase();
result.display = truncateText(body, CONFIG.maxRequestBodyLength);
if (/application\/x-www-form-urlencoded/i.test(normalizedContentType)) {
result.parsed = parseUrlEncodedBody(body);
if (result.parsed && Object.keys(result.parsed).length) {
result.kind = 'urlencoded';
result.display = truncateText(JSON.stringify(result.parsed, null, 2), CONFIG.maxRequestBodyLength);
inferShape(result.parsed, '$', result.shape, 0);
return result;
}
}
try {
result.parsed = JSON.parse(body);
result.kind = 'json';
Expand Down Expand Up @@ -451,9 +501,9 @@
}

function buildCaptureRecord(base) {
var requestBody = summarizeBody(base.requestBody);
var responseBody = summarizeResponse(base.responseText);
var requestContentType = getHeader(base.requestHeaders, 'content-type');
var requestBody = summarizeBody(base.requestBody, requestContentType);
var responseBody = summarizeResponse(base.responseText);
var responseContentType = base.responseContentType || '';
var actionContext = snapshotActionContext();
return {
Expand Down Expand Up @@ -568,30 +618,71 @@
return originalXHRSend.apply(this, arguments);
};

function isRequestLike(value) {
return !!(value && typeof value === 'object' && typeof value.url === 'string');
}

function resolveFetchRequestInfo(input, options) {
var init = options || {};
var requestLike = isRequestLike(input) ? input : null;
var hasOwnBody = Object.prototype.hasOwnProperty.call(init, 'body');
var requestHeaders = mergeHeaders(requestLike ? requestLike.headers : {}, init.headers || {});
var requestBody = hasOwnBody ? init.body : undefined;
var bodyPromise;

if (typeof requestBody !== 'undefined') {
bodyPromise = Promise.resolve(requestBody);
} else if (requestLike && typeof requestLike.clone === 'function') {
try {
bodyPromise = requestLike.clone().text().then(
function(text) {
return text || '';
},
function() {
return '';
}
);
} catch (error) {
bodyPromise = Promise.resolve('');
}
} else {
bodyPromise = Promise.resolve('');
}

return {
method: ((init.method || (requestLike && requestLike.method) || 'GET') + '').toUpperCase(),
requestHeaders: requestHeaders,
requestBodyPromise: bodyPromise,
url: typeof input === 'string' ? input : (input && input.url ? input.url : String(input))
};
}

var originalFetch = window.fetch;
window.fetch = function(url, options) {
options = options || {};
var requestInfo = resolveFetchRequestInfo(url, options);
var startTime = Date.now();
var method = (options.method || 'GET').toUpperCase();
var requestHeaders = normalizeHeaders(options.headers || {});
var urlStr = typeof url === 'string' ? url : (url && url.url ? url.url : String(url));
var decision = getCaptureDecision(urlStr, method, requestHeaders);
var decision = getCaptureDecision(requestInfo.url, requestInfo.method, requestInfo.requestHeaders);

if (!decision.capture) {
return originalFetch.apply(this, arguments);
}

return originalFetch.apply(this, arguments).then(function(response) {
var cloned = response.clone();
return cloned.text().then(function(text) {
return Promise.all([
requestInfo.requestBodyPromise,
cloned.text()
]).then(function(values) {
var requestBody = values[0];
var text = values[1];
var record = buildCaptureRecord({
type: 'Fetch',
method: method,
url: urlStr,
method: requestInfo.method,
url: requestInfo.url,
urlInfo: decision.urlInfo,
status: response.status,
requestHeaders: requestHeaders,
requestBody: options.body,
requestHeaders: requestInfo.requestHeaders,
requestBody: requestBody,
responseText: text || '',
responseContentType: response.headers && typeof response.headers.get === 'function'
? (response.headers.get('content-type') || '')
Expand All @@ -604,7 +695,7 @@
window.__capturedRequests.push(record);
console.log(
'[API Capture] Fetch:',
method,
requestInfo.method,
record.normalizedUrl,
'->',
response.status,
Expand All @@ -615,12 +706,12 @@
}).catch(function(error) {
var record = buildCaptureRecord({
type: 'Fetch',
method: method,
url: urlStr,
method: requestInfo.method,
url: requestInfo.url,
urlInfo: decision.urlInfo,
status: 'error',
requestHeaders: requestHeaders,
requestBody: options.body,
requestHeaders: requestInfo.requestHeaders,
requestBody: '',
responseText: '',
responseContentType: '',
pageContext: getPageContext(),
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ docker run -d \
--name flocks \
-p 8000:8000 \
-p 5173:5173 \
--shm-size 2gb \
--shm-size 4gb \
-v "${HOME}/.flocks:/home/flocks/.flocks" \
ghcr.io/agentflocks/flocks:latest
```
Expand All @@ -153,7 +153,7 @@ docker run -d `
--name flocks `
-p 8000:8000 `
-p 5173:5173 `
--shm-size 2gb `
--shm-size 4gb `
-v "${env:USERPROFILE}\.flocks:/home/flocks/.flocks" `
ghcr.io/agentflocks/flocks:latest
```
Expand Down
4 changes: 2 additions & 2 deletions README_zh.md
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ docker run -d \
-e TZ=Asia/Shanghai \
-p 8000:8000 \
-p 5173:5173 \
--shm-size 2gb \
--shm-size 4gb \
-v "${HOME}/.flocks:/home/flocks/.flocks" \
ghcr.io/agentflocks/flocks:latest
```
Expand All @@ -154,7 +154,7 @@ docker run -d `
-e TZ=Asia/Shanghai `
-p 8000:8000 `
-p 5173:5173 `
--shm-size 2gb `
--shm-size 4gb `
-v "${env:USERPROFILE}\.flocks:/home/flocks/.flocks" `
ghcr.io/agentflocks/flocks:latest
```
Expand Down
2 changes: 1 addition & 1 deletion flocks/browser/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -487,7 +487,7 @@ def row(label: str, ok: bool, detail: str = "") -> None:
browser_running,
"" if browser_running else "start Chrome, Chromium, or Edge and rerun `flocks browser --setup`",
)
row("daemon alive", daemon, "" if daemon else "not running; run `flocks browser --setup` to attach")
row("daemon alive", daemon, "" if daemon else "not running; wait user open browser inspect page then run `flocks browser --setup` to attach")
row("active browser connections", bool(connections), str(len(connections)))
for conn in connections:
page = conn.get("page")
Expand Down
Loading