Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
077a38e
Update for branch-22.08
thomcom Aug 31, 2022
99d76ad
Modify edge mapper to use a table .join, with error handling, instead…
thomcom Sep 1, 2022
7f69939
Remove set_missing_fields_as_nulls workaround.
thomcom Sep 2, 2022
adda810
First bits of template for abstract request handling.
thomcom Sep 2, 2022
e685652
Add routes/gpu
thomcom Sep 2, 2022
5093b86
Add Series.repeat
thomcom Sep 6, 2022
a8f6eb8
Merge branch 'main' into feature/api-server-abstraction
thomcom Oct 5, 2022
c302c01
Fix yarn.lock
thomcom Oct 5, 2022
d4e721f
Clean up and pass a couple of tests.
thomcom Oct 11, 2022
cc3cbab
Adding flexibility to gpu handler.
thomcom Oct 11, 2022
0eca0f4
Fiddling with dynamic dispatch
thomcom Oct 11, 2022
ba4c867
Drop the abstract server.
thomcom Oct 12, 2022
1e1758f
Particles endpoints to send data to client.
thomcom Oct 13, 2022
316a6a6
Fix a couple errors.
thomcom Oct 14, 2022
5288d6c
Refactor particles handler to take two routes.
thomcom Nov 10, 2022
dee37c8
Add lon/lat filtering to request.
thomcom Nov 10, 2022
8e19053
Tweaking for better filtering, memory safety.
thomcom Nov 16, 2022
60c48de
Drop z, w, r, g, b from particles request.
thomcom Nov 17, 2022
e8f6b59
Add logging for timed out currentDataframe.dispose() that causes a 500
thomcom Nov 17, 2022
7e51b88
Pass all tests and add some coverage, particles tests.
thomcom Nov 21, 2022
7401333
Add new test files.
thomcom Nov 21, 2022
4fdc5d7
Write unit tests for particles/get_shader_column in preparation of wr…
thomcom Nov 21, 2022
f28d277
Write npoints test and refactor endpoint.
thomcom Nov 22, 2022
f371cc7
Add quadtree loading route.
thomcom Nov 22, 2022
54d3d31
Add set_polygons_quadtree and refactor gpu_cache.
thomcom Nov 23, 2022
6b2a082
Set poly types.
thomcom Nov 23, 2022
43aef81
Write quadtree/get_points
thomcom Nov 23, 2022
63aee4d
Update modules/demo/api-server/package.json
thomcom Nov 30, 2022
bff76ab
Merge branch 'main' of github.com:rapidsai/node into feature/api-serv…
trxcllnt Nov 30, 2022
6228a61
Update package.json
thomcom Nov 30, 2022
bc7f5bd
Drop repeat
thomcom Nov 30, 2022
8d0f953
Merge branch 'feature/api-server-abstraction' of github.com:thomcom/n…
thomcom Nov 30, 2022
fe8679f
Really important comma needed to go.
thomcom Nov 30, 2022
25ed45c
Merge branch 'feature/api-server-abstraction' of github.com:thomcom/n…
trxcllnt Nov 30, 2022
1be7b71
Update modules/demo/api-server/routes/gpu/index.js
thomcom Jan 4, 2023
f687420
Update modules/demo/api-server/test/routes/graphology.test.js
thomcom Jan 4, 2023
5a5efb5
Update modules/demo/api-server/test/routes/graphology.test.js
thomcom Jan 4, 2023
2d3ccb2
For some reason the route path changed. Upstream changes?
thomcom Jan 3, 2023
7ebef59
Trying to figure out issue with eslint improperly formatting in here.
thomcom Jan 3, 2023
ccb42f8
Reorder exports to make clang-format happy.
thomcom Jan 3, 2023
a537ad2
Writing docs for api-server
thomcom Jan 4, 2023
695832b
Add docs, refactor gpu calls.
thomcom Jan 4, 2023
4c3e0fb
Quadtree tweaks.
thomcom Jan 9, 2023
fa9447c
Add quadtree count endpoint, plus clean up schema.
thomcom Jan 13, 2023
b174028
Endpoint to return just the first n points.
thomcom Jan 17, 2023
6b23188
Next and clear endpoints.
thomcom Jan 18, 2023
230d6a3
Fix bug with error in next endpoint.
thomcom Jan 23, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Write quadtree/get_points
  • Loading branch information
thomcom committed Nov 23, 2022
commit 43aef81408084daab4eedc7feb399f8e5d017206
63 changes: 54 additions & 9 deletions modules/demo/api-server/routes/quadtree/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ module.exports = async function(fastify, opts) {
fastify.register(arrowPlugin);
fastify.decorate('cacheObject', gpu_cache.cacheObject);
fastify.decorate('getData', gpu_cache.getData);
fastify.decorate('listDataframes', gpu_cache.listDataframes);
fastify.decorate('readCSV', gpu_cache.readCSV);

const get_schema = {
Expand Down Expand Up @@ -70,7 +71,7 @@ module.exports = async function(fastify, opts) {
parseFloat(table.get(request.body.yAxisName).max()),
];
try {
const quadtree = cuspatial.Quadtree.new({
const quadtree = cuspatial.Quadtree.new({
x: table.get(request.body.xAxisName),
y: table.get(request.body.yAxisName),
xMin,
Expand All @@ -81,10 +82,12 @@ module.exports = async function(fastify, opts) {
maxDepth: 15,
minSize: 1e5
});
const saved = await fastify.cacheObject(request.params.table);
result.message = 'Quadtree created';
result.success = true;
result.statusCode = 200;
const quadtree_name = request.params.table + '_quadtree';
request.params.quadtree = quadtree_name
const saved = await fastify.cacheObject(quadtree_name, quadtree);
result.message = 'Quadtree created';
result.success = true;
result.statusCode = 200;
await reply.code(result.statusCode).send(result);
} catch (e) {
result.message = e;
Expand All @@ -100,17 +103,22 @@ module.exports = async function(fastify, opts) {
method: 'POST',
url: '/set_polygons_quadtree',
schema: {
querystring:
{polygon_offset: {type: 'array'}, ring_offset: {type: 'array'}, points: {type: 'array'}}
querystring: {
name: {type: 'string'},
polygon_offset: {type: 'array'},
ring_offset: {type: 'array'},
points: {type: 'array'}
}
},
handler: async (request, reply) => {
let message = 'Error';
let result = {'params': request.params, success: false, message: message};
try {
const polygon_offset = Series.new(new Int32Array(request.body.polygon_offset));
const ring_offset = Series.new(new Int32Array(request.body.ring_offset));
const points = Series.new(new Float32Array(request.body.points));
fastify.cacheObject(request.body.name, {polygon_offset, ring_offset, points});
const points = Series.new(new Float64Array(request.body.points));
const cached =
await fastify.cacheObject(request.body.name, {polygon_offset, ring_offset, points});
result.message = 'Set polygon ' + request.body.name;
result.success = true;
result.statusCode = 200;
Expand All @@ -124,4 +132,41 @@ module.exports = async function(fastify, opts) {
}
}
});

fastify.route({
method: 'GET',
url: '/get_points/:quadtree/:polygon',
schema: {querystring: {quadtree: {type: 'string'}, polygon: {type: 'string'}}},
handler: async (request, reply) => {
let message = 'Error';
let result = {'params': request.params, success: false, message: message};
try {
const quadtree = await fastify.getData(request.params.quadtree);
const {polygon_offset, ring_offset, points} = await fastify.getData(request.params.polygon);
const data = await fastify.listDataframes();
const pts = cuspatial.makePoints(
points.gather(Series.sequence({size: points.length, step: 2, init: 0})),
points.gather(Series.sequence({size: points.length, step: 2, init: 1})));
const polylines = cuspatial.makePolylines(pts, ring_offset);
const polygons = cuspatial.makePolygons(polylines, polygon_offset);
const polyPointPairs = quadtree.pointInPolygon(polygons);
const resultPoints = quadtree.points.gather(polyPointPairs.get('point_index'));
const numPoints = resultPoints.get('x').length
let result_col = Series.sequence({size: numPoints * 2, step: 0, init: 0});
result_col = result_col.scatter(resultPoints.get('x'),
Series.sequence({size: numPoints, step: 2, init: 0}));
result_col = result_col.scatter(resultPoints.get('y'),
Series.sequence({size: numPoints, step: 2, init: 1}));
result = new DataFrame({'points_in_polygon': result_col})
const writer = RecordBatchStreamWriter.writeAll(result.toArrow());
writer.close();
await reply.code(200).send(writer.toNodeStream());
} catch (e) {
result.message = e;
result.success = false;
result.statusCode = 500;
await reply.code(result.statusCode).send(result);
}
}
});
}
39 changes: 37 additions & 2 deletions modules/demo/api-server/test/routes/quadtree.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,12 @@ test('quadtree/create/:table', async (t) => {
t.same(result, {
statusCode: 200,
message: 'Quadtree created',
params: {table: 'csv_quadtree.csv'},
params: {table: 'csv_quadtree.csv', quadtree: 'csv_quadtree.csv_quadtree'},
success: true
})
});

test('quadtree/set_polygons', {only: true}, async (t) => {
test('quadtree/set_polygons', async (t) => {
const dir = t.testdir(csv_quadtree);
const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/'));
const app = await build(t);
Expand All @@ -66,3 +66,38 @@ test('quadtree/set_polygons', {only: true}, async (t) => {
success: true
})
});

test('quadtree/get_points', {only: true}, async (t) => {
const dir = t.testdir(csv_quadtree);
const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/'));
const app = await build(t);
gpu_cache._setPathForTesting(rpath);
const load = await app.inject(
{method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_quadtree.csv'}});
const create = await app.inject({
method: 'POST',
url: '/quadtree/create/csv_quadtree.csv',
body: {xAxisName: 'x', yAxisName: 'y'}
});
const quadtree_name = JSON.parse(create.payload).params.quadtree;
const set_poly = await app.inject({
method: 'POST',
url: '/quadtree/set_polygons_quadtree',
body: {
name: 'test',
polygon_offset: [0, 1],
ring_offset: [0, 4],
points: [-2, -2, -2, 2, 2, 2, 2, -2]
}
});
const polygons_name = JSON.parse(set_poly.payload).params.name;
const res = await app.inject({
method: 'GET',
url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name,
})
const release = await app.inject({method: 'POST', url: '/graphology/release'});
const table = tableFromIPC(res.rawPayload);
const got = table.getChild('points_in_polygon').toArray();
const expected = [1, -1, -1, 1, 0, 0];
t.same(expected, got);
});
1 change: 1 addition & 0 deletions modules/demo/api-server/util/gpu_cache.js
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ module.exports = {
async cacheObject(name, data) { cacheObject(name, data); },

async getData(name) { return datasets != null ? datasets[name] : undefined; },
getDataSync(name) { return datasets != null ? datasets[name] : undefined; },

async listDataframes() { return datasets != null ? Object.keys(datasets) : []; },

Expand Down