"""Browse, download and cache network datasets.
Each repository lives in its own subpackage:
* :mod:`netgamut.snap` -- the Stanford Large Network Dataset Collection, 109
datasets. Large, well known, and skewed to social, web and technological
systems.
* :mod:`netgamut.netzschleuder` -- networks.skewed.de, 286 networks tagged by
domain, including the biological, transport and multilayer networks SNAP has
none of.
* :mod:`netgamut.konect` -- KONECT, the Koblenz Network Collection, 300
networks described in one fixed vocabulary of format, edge type and loops.
A network Netzschleuder or SNAP also holds is listed there but used from the
other source here, so nothing is counted twice.
* :mod:`netgamut.ogb` -- Open Graph Benchmark, 12 datasets whose node
categories are *named* rather than left as anonymous codes.
* :mod:`netgamut.pyg` -- the PyTorch Geometric corpus, 99 datasets: citation,
web, social and heterophilous graphs.
* :mod:`netgamut.paths` -- curated path data (flight itineraries, navigation
trails) plus the conversion into second-order state networks for Infomap's
memory model, which no edge list can express.
* :mod:`netgamut.ahorn` -- the Aachen Higher-Order Repository of Networks, 83
hypergraphs and simplicial complexes. Its unit is a *set* of nodes rather than
a pair, so it returns a
:class:`~netgamut.Hypergraph` rather than a ``Network``.
Metadata for every source ships inside the package, so browsing is offline and
instant; only ``load()`` touches the network, and it caches into a shared
user-level directory so a dataset is fetched once per machine.
Beyond the datasets, :mod:`netgamut.generate` samples synthetic benchmark
networks with planted communities from any blueprint in the pool (requires
the ``generate`` extra).
Every network -- downloaded or generated, first-order, state or multilayer --
converts and writes the same way: ``to_networkx()``, ``to_infomap()``, and
``to_pajek(directory)`` / ``to_link_list(directory)``, which return the path
of the file they wrote.
All sources are mapped onto one domain vocabulary, so a comparison can span
kinds of system rather than kinds of provenance::
import netgamut
netgamut.domains()
# {'Social': 201, 'Informational': 68, 'Biological': 36, ...}
for dataset in netgamut.search(domain="Biological", max_nodes=1000):
print(dataset.source, dataset.name)
Please cite the datasets you use -- each source's records carry the reference
its site asks for.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from . import ahorn, generate, konect, netzschleuder, ogb, paths, pyg, snap
from ._core.cache import (
CACHE_ENV_VAR,
CachedDataset,
CacheInfo,
cache_dir,
cache_info,
clear_cache,
set_cache_dir,
)
from ._core.columns import Columns
from ._core.domains import DOMAINS
from ._core.hypergraph import Hypergraph
from ._core.in_memory import InMemoryNetwork
from ._core.multilayer import MultilayerNetwork
from ._core.network import Network
from ._core.state_network import StateNetwork
__all__ = [
"CACHE_ENV_VAR",
"DOMAINS",
"CacheInfo",
"CachedDataset",
"Columns",
"Hypergraph",
"Match",
"MultilayerNetwork",
"Network",
"StateNetwork",
"InMemoryNetwork",
"__version__",
"ahorn",
"cache_dir",
"cache_info",
"clear_cache",
"domains",
"generate",
"konect",
"load",
"netzschleuder",
"ogb",
"paths",
"pyg",
"search",
"set_cache_dir",
"snap",
"sources",
]
__version__ = "0.4.0"
#: Source name -> subpackage, in the order results are reported.
_SOURCES: dict[str, Any] = {
"snap": snap,
"netzschleuder": netzschleuder,
"konect": konect,
"ogb": ogb,
"pyg": pyg,
"paths": paths,
"ahorn": ahorn,
}
#: The kinds of network :class:`Match` can say yes, no or unknown to, which
#: :func:`search`, the explorer's index and the web mirror's manifest share.
_KINDS = ("weighted", "bipartite", "temporal", "multilayer", "signed", "multigraph")
[docs]
def sources() -> tuple[str, ...]:
"""The available data sources."""
return tuple(_SOURCES)
def _shadowed(source: str, dataset: Any) -> bool:
"""Whether cross-source queries skip *dataset* for another source's copy.
Only KONECT defers: a third of it is Netzschleuder's or SNAP's data. See
:func:`netgamut.konect.duplicates`.
"""
return source == "konect" and dataset.duplicate_of is not None
def _owners(name: str) -> list[str]:
"""The sources holding *name*, leaving out a KONECT copy of another's."""
found = [key for key, module in _SOURCES.items() if name in module.catalog()]
if len(found) > 1 and "konect" in found and _shadowed("konect", konect.info(name)):
found.remove("konect")
return found
[docs]
@dataclass(frozen=True, slots=True)
class Match:
"""One dataset found by a cross-source query.
Keeps the source alongside the record, because the two catalogs use
different names for the same idea and a few names appear in both.
"""
source: str
dataset: Any
@property
def name(self) -> str:
name: str = self.dataset.name
return name
@property
def domains(self) -> tuple[str, ...]:
result: tuple[str, ...] = self.dataset.domains
return result
@property
def nodes(self) -> int | None:
return self.dataset.nodes # type: ignore[no-any-return]
@property
def edges(self) -> int | None:
return self.dataset.edges # type: ignore[no-any-return]
@property
def directed(self) -> bool | None:
return self.dataset.directed # type: ignore[no-any-return]
@property
def node_metadata(self) -> bool | None:
"""Whether :meth:`Network.nodes` returns per-node attributes; ``None`` if unknown.
Each source answers from its own catalog, before any download: KONECT and
Netzschleuder from their file listings and analyses, SNAP from its label
files, OGB and PyG from their class labels. AHORN keeps node lines inside
the dataset file, so it cannot say.
"""
value = getattr(self.dataset, "node_metadata", None)
return None if value is None else bool(value)
@property
def node_attributes(self) -> tuple[str, ...] | None:
"""The per-node columns the source's catalog can name, ``None`` if unknown.
Exact for Netzschleuder, KONECT and SNAP. OGB names only its extra files
(``year``, ``species``) and PyG none, since their other column names come
with the download; ``nodes()`` returns those too.
"""
value = getattr(self.dataset, "node_attributes", None)
return None if value is None else tuple(value)
@property
def loops(self) -> bool | None:
"""Whether any node links to itself, where some source has checked.
KONECT checks every network; for another source's network, KONECT's
check is used when it holds the same edge list. ``None`` means unknown.
"""
value = getattr(self.dataset, "loops", None)
if value is not None:
return bool(value)
return konect.loops_for(self.key)
def _flag(self, name: str) -> bool | None:
value = getattr(self.dataset, name, None)
return None if value is None else bool(value)
@property
def weighted(self) -> bool | None:
"""Whether links carry weights; ``None`` where the source cannot say.
From Netzschleuder's tags, KONECT's edge type, SNAP's graph type and
AHORN's catalog. OGB, PyG and the path data cannot say.
"""
return self._flag("weighted")
@property
def bipartite(self) -> bool | None:
"""Whether links only join two kinds of node; ``None`` where the source cannot say.
KONECT and SNAP say so in their catalogs. For Netzschleuder it is the
site's structural check, no odd cycle, which every two-mode network
passes and so do a few trees and grids.
"""
return self._flag("bipartite")
@property
def temporal(self) -> bool | None:
"""Whether links carry timestamps; ``None`` where the source cannot say.
From Netzschleuder's ``Temporal`` and ``Timestamps`` tags, KONECT's
timestamp column, SNAP's graph type and AHORN's catalog.
"""
return self._flag("temporal")
@property
def multilayer(self) -> bool | None:
"""Whether Netzschleuder publishes it as a multilayer or multiplex network.
``None`` for every other source, whose catalogs do not say.
"""
return self._flag("multilayer")
@property
def signed(self) -> bool | None:
"""Whether links are positive or negative; ``None`` where the source cannot say.
From Netzschleuder's ``Signed`` tag, KONECT's edge type and SNAP's graph type.
"""
return self._flag("signed")
@property
def multigraph(self) -> bool | None:
"""Whether some pair of nodes links more than once; ``None`` if the source cannot say.
From Netzschleuder's ``Multigraph`` tag and KONECT's edge type.
"""
return self._flag("multigraph")
@property
def key(self) -> str:
"""``"<source>/<name>"`` -- unique across sources."""
return f"{self.source}/{self.name}"
[docs]
def load(self, **kwargs: Any) -> Network:
"""Download this dataset via its own source."""
result: Network = _SOURCES[self.source].load(self.name, **kwargs)
return result
[docs]
def to_dict(self) -> dict[str, Any]:
"""A JSON-serialisable summary, the same shape for every source.
Holds the fields :func:`search` filters on plus what a reader needs to
choose and cite a dataset: its description, citations, its page at the
source (``url``) and, where the catalog records it, the original
publisher (``upstream_url``). ``weighted``, ``bipartite``, ``temporal``,
``multilayer``, ``signed`` and ``multigraph`` are the properties of the
same names, ``None`` where the source cannot say. ``duplicate_of`` is
the key of the network a KONECT copy defers to (see
:func:`netgamut.konect.duplicates`), else ``None``.
``subgraph_count`` is how many networks the name holds: a Netzschleuder
entry can hold several, the first few named in ``subgraphs``, and the
statistics describe the first, which :meth:`load` fetches by default.
Other sources hold one and name none.
"""
dataset = self.dataset
attributes = self.node_attributes
upstream = getattr(dataset, "upstream_url", "") or getattr(dataset, "source_url", "")
return {
"key": self.key,
"source": self.source,
"name": self.name,
"title": getattr(dataset, "title", ""),
"domains": list(self.domains),
"nodes": self.nodes,
"edges": self.edges,
"directed": self.directed,
"loops": self.loops,
"node_metadata": self.node_metadata,
"node_attributes": None if attributes is None else list(attributes),
**{name: getattr(self, name) for name in _KINDS},
"subgraph_count": getattr(dataset, "subgraph_count", 1),
"subgraphs": list(getattr(dataset, "subgraphs", ())),
"description": getattr(dataset, "description", ""),
"citations": [_citation(entry) for entry in getattr(dataset, "citations", ())],
"url": dataset.url,
"upstream_url": upstream,
"duplicate_of": getattr(dataset, "duplicate_of", None),
}
def __str__(self) -> str:
shape = []
if self.nodes is not None:
shape.append(f"{self.nodes:,} nodes")
if self.edges is not None:
shape.append(f"{self.edges:,} edges")
domains = "/".join(self.domains) or "uncategorised"
tail = f" ({', '.join(shape)})" if shape else ""
return f"{self.key} [{domains}]{tail}"
def _citation(entry: Any) -> dict[str, str]:
"""One citation as ``{"text", "url"}``: Netzschleuder pairs each text with a link."""
if isinstance(entry, str):
return {"text": entry, "url": ""}
text, *link = entry
return {"text": str(text), "url": str(link[0]) if link else ""}
[docs]
def domains() -> dict[str, int]:
"""Dataset counts per domain, summed over every source.
A dataset listed under two domains counts once in each, and a network held
by two sources counts once.
"""
counts = dict.fromkeys(DOMAINS, 0)
for source, module in _SOURCES.items():
for dataset in module.datasets():
if _shadowed(source, dataset):
continue
for domain in dataset.domains:
counts[domain] += 1
return counts
[docs]
def search(
query: str = "",
*,
domain: str | None = None,
source: str | None = None,
directed: bool | None = None,
max_nodes: int | None = None,
min_nodes: int | None = None,
max_edges: int | None = None,
min_edges: int | None = None,
loops: bool | None = None,
node_metadata: bool | None = None,
node_attribute: str | None = None,
weighted: bool | None = None,
bipartite: bool | None = None,
temporal: bool | None = None,
multilayer: bool | None = None,
signed: bool | None = None,
multigraph: bool | None = None,
unknown: bool = False,
) -> tuple[Match, ...]:
"""Search every source at once, filtering on the shared domain vocabulary.
Only the fields every source agrees on are accepted here. For anything
source-specific -- SNAP's ground-truth partitions, Netzschleuder's subtypes,
OGB's loadability, PyG's class -- use that source's own ``search()``.
Size filters exclude datasets whose size the source does not publish rather than
assuming anything: most PyG entries and the largest OGB ones have no counts.
``loops`` does the same with datasets no source has checked for self-links
(see :attr:`Match.loops`), and ``node_metadata`` with datasets whose catalog
cannot say (see :attr:`Match.node_metadata`), and so do ``weighted``,
``bipartite``, ``temporal``, ``multilayer``, ``signed`` and ``multigraph``
(see :attr:`Match.weighted` and the rest). With ``unknown=True`` a dataset
passes every filter its source cannot answer instead, so
``search(weighted=False, unknown=True)`` keeps the networks no catalog calls
weighted. ``node_attribute`` finds
datasets with a per-node column of that name, case-insensitively, among the
names each catalog knows -- ``node_attribute="gender"`` for something to score
a partition against.
A network held by both KONECT and Netzschleuder or SNAP appears once, from
the other source; ``source="konect"`` lists KONECT's copy too.
Raises:
KeyError: Unknown *domain* or *source*.
"""
if domain is not None and domain not in DOMAINS:
raise KeyError(f"Unknown domain {domain!r}. Available: {', '.join(DOMAINS)}")
if source is not None and source not in _SOURCES:
raise KeyError(f"Unknown source {source!r}. Available: {', '.join(_SOURCES)}")
needle = query.casefold().strip()
results: list[Match] = []
wanted = (weighted, bipartite, temporal, multilayer, signed, multigraph)
kinds = dict(zip(_KINDS, wanted, strict=True))
def within(value: int | None, low: int | None, high: int | None) -> bool:
if low is None and high is None:
return True
if value is None:
return unknown
return (low is None or value >= low) and (high is None or value <= high)
def says(value: bool | None, want: bool | None) -> bool:
if want is None:
return True
return unknown if value is None else value is want
for name, module in _SOURCES.items():
if source is not None and name != source:
continue
for dataset in module.datasets():
if source is None and _shadowed(name, dataset):
continue
if domain is not None and domain not in dataset.domains:
continue
text = f"{dataset.name} {getattr(dataset, 'description', '')}".casefold()
if needle and needle not in text:
continue
if not says(dataset.directed, directed):
continue
if not within(dataset.nodes, min_nodes, max_nodes):
continue
if not within(dataset.edges, min_edges, max_edges):
continue
match = Match(source=name, dataset=dataset)
if not says(match.loops, loops) or not says(match.node_metadata, node_metadata):
continue
if not all(says(getattr(match, name), want) for name, want in kinds.items()):
continue
columns = match.node_attributes
if node_attribute is not None and not (
unknown
if columns is None
else any(column.casefold() == node_attribute.casefold() for column in columns)
):
continue
results.append(match)
return tuple(results)
[docs]
def load(name: str, *, source: str | None = None, **kwargs: Any) -> Network:
"""Download a dataset by name, searching every source.
Pass *source* to disambiguate. Without it, a name present in more than one
source is an error rather than an arbitrary choice -- except that KONECT's
copy of another source's network defers to that source.
Raises:
KeyError: No source has *name*, or several do and *source* was omitted.
"""
if source is not None:
if source not in _SOURCES:
raise KeyError(f"Unknown source {source!r}. Available: {', '.join(_SOURCES)}")
result: Network = _SOURCES[source].load(name, **kwargs)
return result
found = _owners(name)
if not found:
raise KeyError(f"No dataset named {name!r} in any source: {', '.join(_SOURCES)}.")
if len(found) > 1:
raise KeyError(
f"{name!r} exists in several sources ({', '.join(found)}). Pass source= to choose."
)
loaded: Network = _SOURCES[found[0]].load(name, **kwargs)
return loaded