FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

perf(storage): stream container listings to avoid O(children) memory by jeswr · Pull Request #2211 · CommunitySolidServer/CommunitySolidServer · GitHub

Open
3 changes: 3 additions & 0 deletions .github/workflows/npm-test.yml
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ jobs:
run: npm run test:ts
- name: Run unit tests
run: npm run test:unit
- name: Run container-listing memory regression test
if: matrix.operating-system == 'ubuntu-latest' && matrix.node-version == '20.x'
run: npm run test:memory
- name: Submit unit test coverage
uses: coverallsapp/github-action@master
with:
Expand Down
1 change: 1 addition & 0 deletions package.json
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
"test:deploy": "test/deploy/validate-configs.sh",
"test:ts": "tsc -p test --noEmit",
"test:integration": "jest test/integration",
"test:memory": "node --expose-gc --max-old-space-size=128 test/memory/container-listing.js",
"test:unit": "jest --config=./jest.coverage.config.js test/unit",
"test:watch": "jest --coverageReporters none --watch test/unit",
"validate": "componentsjs-compile-config urn:solid-server:default:Initializer -c config/default.json -f > /dev/null",
Expand Down
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,8 @@ export class AllowAcceptHeaderWriter extends MetadataWriter {
}

const isStorage = metadata.has(RDF.terms.type, PIM.terms.Storage);
const isEmpty = !metadata.has(LDP.terms.contains);
const empty = metadata.get(SOLID_META.terms.containerEmpty, SOLID_META.terms.ResponseMetadata);
const isEmpty = empty ? empty.value === 'true' : !metadata.has(LDP.terms.contains);
return !isStorage && isEmpty;
}

Expand Down
12 changes: 3 additions & 9 deletions src/init/migration/SingleContainerJsonStorage.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { JsonResourceStorage } from '../../storage/keyvalue/JsonResourceStorage'
import { createErrorMessage } from '../../util/errors/ErrorUtil';
import { isContainerIdentifier } from '../../util/PathUtil';
import { readableToString } from '../../util/StreamUtil';
import { LDP } from '../../util/Vocabularies';

/**
* A variant of a {@link JsonResourceStorage} where the `entries()` call
Expand All @@ -20,12 +19,7 @@ export class SingleContainerJsonStorage<T> extends JsonResourceStorage<T> {
return;
}

// Only need the metadata
container.data.destroy();
const members = container.metadata.getAll(LDP.terms.contains).map((term): string => term.value);

for (const path of members) {
const documentId = { path };
for await (const documentId of this.getContainedResourceIdentifiers(containerId, container)) {
if (isContainerIdentifier(documentId)) {
continue;
}
Expand All @@ -40,8 +34,8 @@ export class SingleContainerJsonStorage<T> extends JsonResourceStorage<T> {
const json = JSON.parse(await readableToString(document.data)) as T;
yield [ key, json ];
} catch (error: unknown) {
this.logger.error(`Unable to parse ${path}. You should probably delete this resource manually. Error: ${
createErrorMessage(error)}`);
this.logger.error(`Unable to parse ${documentId.path}. You should probably delete this resource manually. ` +
`Error: ${createErrorMessage(error)}`);
}
}
}
Expand Down
102 changes: 74 additions & 28 deletions src/storage/DataAccessorBasedStore.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { randomUUID } from 'node:crypto';
import { Readable } from 'node:stream';
import type { NamedNode, Quad, Term } from '@rdfjs/types';
import arrayifyStream from 'arrayify-stream';
import { DataFactory } from 'n3';
Expand Down Expand Up @@ -118,7 +119,6 @@ export class DataAccessorBasedStore implements ResourceStore {

// In the future we want to use getNormalizedMetadata and redirect in case the identifier differs
let metadata = await this.accessor.getMetadata(identifier);
let representation: Representation;

// Potentially add auxiliary related metadata
// Solid, §4.3: "Clients can discover auxiliary resources associated with a subject resource by making an HTTP HEAD
Expand All @@ -127,40 +127,86 @@ export class DataAccessorBasedStore implements ResourceStore {
await this.auxiliaryStrategy.addMetadata(metadata);

const isContainer = isContainerPath(metadata.identifier.value);
let data = metadata.quads();
if (isContainer || isMetadata) {
if (isContainer) {
// Add containment triples of non-auxiliary resources
for await (const child of this.accessor.getChildren(identifier)) {
if (!this.auxiliaryStrategy.isAuxiliaryIdentifier({ path: child.identifier.value })) {
if (!isMetadata) {
metadata.addQuads(child.quads());
}
metadata.add(LDP.terms.contains, child.identifier as NamedNode, SOLID_META.terms.ResponseMetadata);
}

if (!isContainer && !isMetadata) {
return new BasicRepresentation(await this.accessor.getData(identifier), metadata);
}

if (isContainer && isMetadata) {
for await (const child of this.accessor.getChildren(identifier)) {
if (!this.isAuxiliaryResourceMetadata(child)) {
metadata.add(LDP.terms.contains, child.identifier as NamedNode, SOLID_META.terms.ResponseMetadata);
}
data = metadata.quads();
}
}

if (isMetadata) {
metadata = new RepresentationMetadata(this.metadataStrategy.getAuxiliaryIdentifier(identifier));
addResourceMetadata(metadata, false);
metadata.add(RDF.terms.type, SOLID_META.terms.DescriptionResource);
}
const data = isMetadata ? metadata.quads() : await this.streamContainerRepresentation(identifier, metadata);
if (isMetadata) {
metadata = new RepresentationMetadata(this.metadataStrategy.getAuxiliaryIdentifier(identifier));
addResourceMetadata(metadata, false);
metadata.add(RDF.terms.type, SOLID_META.terms.DescriptionResource);
}

metadata.addQuad(DC.terms.namespace, PREFERRED_PREFIX_TERM, 'dc', SOLID_META.terms.ResponseMetadata);
metadata.addQuad(LDP.terms.namespace, PREFERRED_PREFIX_TERM, 'ldp', SOLID_META.terms.ResponseMetadata);
metadata.addQuad(POSIX.terms.namespace, PREFERRED_PREFIX_TERM, 'posix', SOLID_META.terms.ResponseMetadata);
metadata.addQuad(XSD.terms.namespace, PREFERRED_PREFIX_TERM, 'xsd', SOLID_META.terms.ResponseMetadata);
return new BasicRepresentation(data, metadata, INTERNAL_QUADS);
}

/**
* Creates a lazy quad stream for a container listing.
*/
protected async streamContainerRepresentation(identifier: ResourceIdentifier, metadata: RepresentationMetadata):

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

This function seems overly complicated? I might be missing something, but what is the reason you can not just have a function that yields the metadata quads, and then yields all the container listing quads (with perhaps an exception for the first element because of the boolean) without all the code overhead below?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

I've cleaned this up as much as I can in 059b2ae.

The main complexity is that childQuads is initialised to determine containerEmpty; so we need to close it - this is done through the return in the finally.

Promise<Readable> {
const ownQuads = metadata.quads();
const childQuads = this.getContainerListingQuads(identifier, metadata.identifier as NamedNode);
const first = await childQuads.next();
metadata.add(
SOLID_META.terms.containerEmpty,
DataFactory.literal(`${first.done}`, XSD.terms.boolean),
SOLID_META.terms.ResponseMetadata,
);

metadata.addQuad(DC.terms.namespace, PREFERRED_PREFIX_TERM, 'dc', SOLID_META.terms.ResponseMetadata);
metadata.addQuad(LDP.terms.namespace, PREFERRED_PREFIX_TERM, 'ldp', SOLID_META.terms.ResponseMetadata);
metadata.addQuad(POSIX.terms.namespace, PREFERRED_PREFIX_TERM, 'posix', SOLID_META.terms.ResponseMetadata);
metadata.addQuad(XSD.terms.namespace, PREFERRED_PREFIX_TERM, 'xsd', SOLID_META.terms.ResponseMetadata);
async function* generate(): AsyncGenerator<Quad, void, undefined> {
try {
yield* ownQuads;
if (!first.done) {
yield first.value;
yield* childQuads;
}
} finally {
await childQuads.return?.();
}
}
const listing = generate();
// Prime the generator so cancellation reaches its finally block.
const initial = await listing.next();
const data = Readable.from(listing, { objectMode: true });
if (!initial.done) {
data.unshift(initial.value);
}
return data;
}

if (isContainer || isMetadata) {
representation = new BasicRepresentation(data, metadata, INTERNAL_QUADS);
} else {
representation = new BasicRepresentation(await this.accessor.getData(identifier), metadata);
/** Yields containment and metadata quads for non-auxiliary children. */
private async* getContainerListingQuads(identifier: ResourceIdentifier, containerNode: NamedNode):
AsyncIterableIterator<Quad> {
for await (const child of this.accessor.getChildren(identifier)) {
if (!this.isAuxiliaryResourceMetadata(child)) {
yield DataFactory.quad(
containerNode,
LDP.terms.contains,
child.identifier as NamedNode,
SOLID_META.terms.ResponseMetadata,
);
yield* child.quads();
}
}
}

return representation;
private isAuxiliaryResourceMetadata(metadata: RepresentationMetadata): boolean {
return this.auxiliaryStrategy.isAuxiliaryIdentifier({ path: metadata.identifier.value });
}

public async addResource(container: ResourceIdentifier, representation: Representation, conditions?: Conditions):
Expand Down Expand Up @@ -664,7 +710,7 @@ export class DataAccessorBasedStore implements ResourceStore {
*/
protected async hasProperChildren(container: ResourceIdentifier): Promise<boolean> {
for await (const child of this.accessor.getChildren(container)) {
if (!this.auxiliaryStrategy.isAuxiliaryIdentifier({ path: child.identifier.value })) {
if (!this.isAuxiliaryResourceMetadata(child)) {
return true;
}
}
Expand Down
32 changes: 16 additions & 16 deletions src/storage/accessors/InMemoryDataAccessor.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,20 @@ interface DataEntry {
metadata: RepresentationMetadata;
}
interface ContainerEntry {
entries: Record<string, CacheEntry>;
entries: Map<string, CacheEntry>;
metadata: RepresentationMetadata;
}
type CacheEntry = DataEntry | ContainerEntry;

export class InMemoryDataAccessor implements DataAccessor, SingleThreaded {
private readonly identifierStrategy: IdentifierStrategy;
// A dummy container where every entry corresponds to a root container
private readonly store: { entries: Record<string, ContainerEntry> };
private readonly store: { entries: Map<string, CacheEntry> };

public constructor(identifierStrategy: IdentifierStrategy) {
this.identifierStrategy = identifierStrategy;

this.store = { entries: {}};
this.store = { entries: new Map() };
}

public async canHandle(): Promise<void> {
Expand All @@ -54,11 +54,11 @@ export class InMemoryDataAccessor implements DataAccessor, SingleThreaded {
public async* getChildren(identifier: ResourceIdentifier): AsyncIterableIterator<RepresentationMetadata> {
const entry = this.getEntry(identifier);
if (!this.isDataEntry(entry)) {
yield* Object.entries(entry.entries).map(([ path, child ]): RepresentationMetadata => {
for (const [ path, child ] of entry.entries) {
const metadata = new RepresentationMetadata(DataFactory.namedNode(path));
metadata.addQuads(child.metadata.quads());
return metadata;
});
yield metadata;
}
}
}

Expand All @@ -74,10 +74,10 @@ export class InMemoryDataAccessor implements DataAccessor, SingleThreaded {
metadata.set(POSIX.terms.size, `${size}`);
}

parent.entries[identifier.path] = {
parent.entries.set(identifier.path, {
data: dataArray,
metadata,
};
});
}

public async writeContainer(identifier: ResourceIdentifier, metadata: RepresentationMetadata): Promise<void> {
Expand All @@ -89,10 +89,10 @@ export class InMemoryDataAccessor implements DataAccessor, SingleThreaded {
// Create new entry if it didn't exist yet
if (NotFoundHttpError.isInstance(error)) {
const parent = this.getParentEntry(identifier);
parent.entries[identifier.path] = {
entries: {},
parent.entries.set(identifier.path, {
entries: new Map(),
metadata,
};
});
} else {
throw error;
}
Expand All @@ -106,10 +106,9 @@ export class InMemoryDataAccessor implements DataAccessor, SingleThreaded {

public async deleteResource(identifier: ResourceIdentifier): Promise<void> {
const parent = this.getParentEntry(identifier);
if (!parent.entries[identifier.path]) {
if (!parent.entries.delete(identifier.path)) {
throw new NotFoundHttpError();
}
delete parent.entries[identifier.path];
}

private isDataEntry(entry: CacheEntry): entry is DataEntry {
Expand Down Expand Up @@ -142,10 +141,11 @@ export class InMemoryDataAccessor implements DataAccessor, SingleThreaded {

const hierarchy = this.getHierarchy(this.identifierStrategy.getParentContainer(identifier));
for (const entry of hierarchy) {
parent = parent.entries[entry.path];
if (!parent) {
const child: CacheEntry | undefined = parent.entries.get(entry.path);
if (!child) {
throw new NotFoundHttpError();
}
parent = child;
if (this.isDataEntry(parent)) {
throw new InternalServerError('Invalid path.');
}
Expand All @@ -160,7 +160,7 @@ export class InMemoryDataAccessor implements DataAccessor, SingleThreaded {
*/
private getEntry(identifier: ResourceIdentifier): CacheEntry {
const parent = this.getParentEntry(identifier);
const entry = parent.entries[identifier.path];
const entry = parent.entries.get(identifier.path);
if (!entry) {
throw new NotFoundHttpError();
}
Expand Down
22 changes: 17 additions & 5 deletions src/storage/keyvalue/JsonResourceStorage.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { Quad } from '@rdfjs/types';
import { BasicRepresentation } from '../../http/representation/BasicRepresentation';
import type { Representation } from '../../http/representation/Representation';
import type { ResourceIdentifier } from '../../http/representation/ResourceIdentifier';
Expand Down Expand Up @@ -83,11 +84,8 @@ export class JsonResourceStorage<T> implements KeyValueStorage<string, T> {
const representation = await this.safelyGetResource(identifier);
if (representation) {
if (isContainerIdentifier(identifier)) {
// Only need the metadata
representation.data.destroy();
const members = representation.metadata.getAll(LDP.terms.contains).map((term): string => term.value);
for (const path of members) {
yield* this.getResourceEntries({ path });
for await (const member of this.getContainedResourceIdentifiers(identifier, representation)) {
yield* this.getResourceEntries(member);
}
} else {
try {
Expand All @@ -102,6 +100,20 @@ export class JsonResourceStorage<T> implements KeyValueStorage<string, T> {
}
}

/** Streams direct member identifiers from a container representation. */
protected async* getContainedResourceIdentifiers(
identifier: ResourceIdentifier,
representation: Representation,
): AsyncIterableIterator<ResourceIdentifier> {
for await (const entry of representation.data as AsyncIterable<Partial<Quad>>) {
if (entry.subject?.termType === 'NamedNode' && entry.subject.value === identifier.path &&
entry.predicate?.termType === 'NamedNode' && entry.predicate.value === LDP.terms.contains.value &&
entry.object?.termType === 'NamedNode') {
yield { path: entry.object.value };
}
}
}

/**
* Returns the representation for the given identifier.
* Returns undefined if a 404 error is thrown.
Expand Down
3 changes: 3 additions & 0 deletions src/util/Vocabularies.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,8 @@ export const SOLID_META = createVocabulary(
'value',
// This is used to indicate whether metadata should be preserved or not during a PUT operation
'preserve',
// Indicates whether a container has no contained resources
'containerEmpty',
// These predicates are used to describe the requested access in case of an unauthorized request
'requestedAccess',
'accessTarget',
Expand All @@ -341,6 +343,7 @@ export const VCARD = createVocabulary(

export const XSD = createVocabulary(
'http://www.w3.org/2001/XMLSchema#',
'boolean',
'dateTime',
'duration',
'integer',
Expand Down
Loading
Loading

Back | FazBrowse Home | New Git URL