v/ln-stream
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add local snapshot to repo; add Dockerfile; update docker-compose.yml;

Browse source
This commit is contained in:
davisv7 2025-08-31 15:56:34 -04:00
parent bc74ef9cbd
commit d55d7832bc
10 changed files with 2471532 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,3 +1,4 @@
.DS_Store .DS_Store
.env .env
.idea/ .idea/
/creds

22
Dockerfile Normal file
View file

@ -0,0 +1,22 @@
# Use the official Golang image to create a build artifact.
FROM golang:1.25
# Set the Current Working Directory inside the container
WORKDIR /app
# Copy go mod and sum files
COPY go.mod go.sum ./
#COPY .env ./
COPY describegraph.json /app/describegraph.json
# Download all dependencies. Dependencies will be cached if the go.mod and go.sum files are not changed
RUN go mod tidy
# Copy the source from the current directory to the Working Directory inside the container
COPY . .
# Build the Go app
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o lnstream .
# Command to run the executable
CMD ["./lnstream"]

2471271
describegraph.json Normal file
View file

File diff suppressed because it is too large Load diff

42
docker-compose.yml
View file

@ -5,3 +5,45 @@ services:
ports: ports:
- '7687:7687' - '7687:7687'
- '7444:7444' - '7444:7444'
networks:
- lnstream-network
memgraph-lab:
image: memgraph/lab
environment:
- QUICK_CONNECT_MG_HOST=host.docker.internal
ports:
- '3000:3000'
networks:
- lnstream-network
api:
build: .
ports:
- '8080:8080'
depends_on:
- memgraph-mage
# - postgres
environment:
- NEO4J_HOST=memgraph-mage # Assuming your application uses this environment variable
- NEO4J_PORT=7687 # Assuming your application uses this environment variable
- NEO4J_USERNAME=
- NEO4J_PASSWORD=
# volumes:
# - ${LND_CRED_PATH}:/app/creds
networks:
- lnstream-network
# postgres:
# image: postgres:latest
# environment:
# POSTGRES_USER: user
# POSTGRES_PASSWORD: password
# POSTGRES_DB: lnstreamdb
# ports:
# - "5432:5432"
# networks:
# - lnstream-network
networks:
lnstream-network:
driver: bridge

1
index.html
View file

@ -7,6 +7,7 @@
<body> <body>
<button id="resetButton">Reset Graph in Neo4j</button> <button id="resetButton">Reset Graph in Neo4j</button>
<button id="toggleButton">Toggle Graph Updates</button> <button id="toggleButton">Toggle Graph Updates</button>
<button id="localButton">Load Local Snapshot</button>
<span id="indicator"></span> <span id="indicator"></span>
<script src="/static/script.js"></script> <script src="/static/script.js"></script>
</body> </body>

122
lnd/lnd.go
View file

@ -2,9 +2,11 @@ package lnd
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"github.com/lightninglabs/lndclient" "github.com/lightninglabs/lndclient"
"github.com/neo4j/neo4j-go-driver/v4/neo4j" "github.com/neo4j/neo4j-go-driver/v4/neo4j"
"io"
"log" "log"
"os" "os"
"strings" "strings"
@ -28,6 +30,56 @@ func ConnectToLND() (*lndclient.GrpcLndServices, error) {
return lndclient.NewLndServices(&config) return lndclient.NewLndServices(&config)
} }
type Node struct {
Pub_Key string
LastUpdate time.Time
Alias string
Color string
Features map[string]interface{}
Addresses []interface{}
}
type ChannelEdge struct {
ChannelId uint64 `json:"channel_id,string"`
Capacity string `json:"capacity"`
Node1_Pub string `json:"node1_pub"`
Node2_Pub string `json:"node2_pub"`
Node1Policy RoutingPolicy `json:"node1_policy,omitempty"`
Node2Policy RoutingPolicy `json:"node2_policy,omitempty"`
}
// RoutingPolicy holds the edge routing policy for a channel edge.
type RoutingPolicy struct {
TimeLockDelta int `json:"time_lock_delta"`
MinHtlc string `json:"min_htlc"`
FeeBaseMsat string `json:"fee_base_msat"`
FeeRateMilliMsat string `json:"fee_rate_milli_msat"`
Disabled bool `json:"disabled"`
MaxHtlcMsat string `json:"max_htlc_msat"`
LastUpdate int `json:"last_update"`
CustomRecords struct {
} `json:"custom_records"`
}
// Graph describes our view of the graph.
type Graph struct {
// Nodes is the set of nodes in the channel graph.
Nodes []Node
// Edges is the set of edges in the channel graph.
Edges []ChannelEdge
}
func writeNodesToMemgraph(session neo4j.Session, nodes []lndclient.Node) { func writeNodesToMemgraph(session neo4j.Session, nodes []lndclient.Node) {
const batchSize = 100 const batchSize = 100
@ -193,3 +245,73 @@ func WriteGraphToMemgraph(graph *lndclient.Graph, neo4jDriver neo4j.Driver) {
writeChannelsToMemgraph(session, graph.Edges) writeChannelsToMemgraph(session, graph.Edges)
fmt.Println("Finished writing to Memgraph...") fmt.Println("Finished writing to Memgraph...")
} }
func WriteSnapshotToMemgraph(snapshotFilename string, neo4jDriver neo4j.Driver) {
var err error
session := neo4jDriver.NewSession(neo4j.SessionConfig{})
defer session.Close()
if err != nil {
log.Fatalf("Failed to connect to memgraph: %v", err)
}
jsonFile, err := os.Open(snapshotFilename)
if err != nil {
log.Fatalf("Failed to open snapshot: %v", err)
}
defer jsonFile.Close()
byteValue, _ := io.ReadAll(jsonFile)
var graph Graph
err = json.Unmarshal(byteValue, &graph)
if err != nil {
log.Fatalf("Failed to open snapshot: %v", err)
}
fmt.Println("Writing to Memgraph...")
createNodeIndex(session)
createIndexForChannels(session)
writeSnapshotNodesToMemgraph(session, graph.Nodes)
writeSnapshotChannelsToMemgraph(session, graph.Edges)
fmt.Println("Finished writing to Memgraph...")
}
func writeSnapshotNodesToMemgraph(session neo4j.Session, nodes []Node) {
for _, node := range nodes {
_, is_wumbo := node.Features["19"]
query := "MERGE (n:node {pubkey: $pubKey, alias: $alias, is_wumbo: $is_wumbo})"
params := map[string]interface{}{
"pubKey": node.Pub_Key,
"alias": node.Alias,
"is_wumbo": is_wumbo,
}
_, err := session.Run(query, params)
if err != nil {
log.Printf("Failed to execute node query: %v", err)
}
}
}
func writeSnapshotChannelsToMemgraph(session neo4j.Session, edges []ChannelEdge) {
for _, edge := range edges {
chanID := convertChannelIDToString(edge.ChannelId) // Convert uint64 to string format
writeChannelPolicyToMemgraphSnapshot(session, &edge, edge.Node1Policy, edge.Node1_Pub, edge.Node2_Pub, chanID)
writeChannelPolicyToMemgraphSnapshot(session, &edge, edge.Node2Policy, edge.Node2_Pub, edge.Node1_Pub, chanID)
}
}
func writeChannelPolicyToMemgraphSnapshot(session neo4j.Session, edge *ChannelEdge, policy RoutingPolicy, node1PubKey, node2PubKey, chanID string) {
if policy.MaxHtlcMsat != "" {
query := fmt.Sprintf(`
MATCH (a:node {pubkey: '%s'}), (b:node {pubkey: '%s'})
MERGE (a)-[r:CHANNEL {channel_id: '%s', capacity: %s}]->(b)
SET r.fee_base_msat = %s, r.fee_rate_milli_msat = %s, r.time_lock_delta = %d,
r.disabled = %v, r.min_htlc_msat = %s, r.max_htlc_msat = %s
`, node1PubKey, node2PubKey, chanID, edge.Capacity,
policy.FeeBaseMsat, policy.FeeRateMilliMsat, policy.TimeLockDelta, policy.Disabled, policy.MinHtlc, policy.MaxHtlcMsat)
_, err := session.Run(query, nil)
if err != nil {
log.Printf("Failed to execute channel policy query: %v", err)
}
}
}

1
main.go
View file

@ -34,6 +34,7 @@ func main() {
router := gin.Default() router := gin.Default()
// Handle the button click // Handle the button click
router.GET("/reset-graph", routes.ResetGraphHandler) router.GET("/reset-graph", routes.ResetGraphHandler)
router.GET("/load-local-snapshot", routes.LoadLocalSnapshot)
// Handle the button click to toggle the routine // Handle the button click to toggle the routine
router.GET("/toggle-updates", routes.ToggleUpdatesHandler) router.GET("/toggle-updates", routes.ToggleUpdatesHandler)

42
memgraph/memgraph.go
View file

@ -10,7 +10,9 @@ import (
// ConnectNeo4j creates a new Neo4j driver instance and establishes a connection // ConnectNeo4j creates a new Neo4j driver instance and establishes a connection
func ConnectNeo4j() (neo4j.Driver, error) { func ConnectNeo4j() (neo4j.Driver, error) {
uri := os.Getenv("NEO4J_URI") host := os.Getenv("NEO4J_HOST")
port := os.Getenv("NEO4J_PORT")
uri := "bolt://" + host + ":" + port
username := os.Getenv("NEO4J_USERNAME") username := os.Getenv("NEO4J_USERNAME")
password := os.Getenv("NEO4J_PASSWORD") password := os.Getenv("NEO4J_PASSWORD")
@ -141,3 +143,41 @@ func ProcessUpdates(driver neo4j.Driver, update *lndclient.GraphTopologyUpdate)
} }
} }
} }
// SetupAfterImport runs a set of commands directly after importing a snapshot.
func SetupAfterImport(neo4jDriver neo4j.Driver) {
fmt.Println("SetupAfterImport")
session := neo4jDriver.NewSession(neo4j.SessionConfig{})
defer session.Close()
// fix denominations
_, err := session.Run("match (n)-[r]->(m)\nset r.fee_base_milli_msat = r.fee_base_msat*1000", nil)
if err != nil {
log.Printf("Failed to run command: %v", err)
}
// set node capacity
_, err = session.Run("match (n)\nset n.total_capacity = 0;\n", nil)
if err != nil {
log.Printf("Failed to run command: %v", err)
}
_, err = session.Run("MATCH (n)-[r]-(m)\nWITH n,sum(r.capacity) as total_capacity\nSET n.total_capacity = total_capacity/2;", nil)
if err != nil {
log.Printf("Failed to run command: %v", err)
}
// set node betweeness
_, err = session.Run("call betweenness_centrality.get() YIELD betweenness_centrality, node \nwith betweenness_centrality,node\nset node.betweenness_centrality = betweenness_centrality;", nil)
if err != nil {
log.Printf("Failed to run command: %v", err)
}
// set edge betweenness
_, err = session.Run("MATCH (n)-[r]-(m)\nset r.betweenness_centrality = (n.betweenness_centrality+m.betweenness_centrality)/2;", nil)
if err != nil {
log.Printf("Failed to run command: %v", err)
}
// set fee ratios
_, err = session.Run("MATCH (n)-[r]-(m)\nset r.betweenness_centrality = (n.betweenness_centrality+m.betweenness_centrality)/2;", nil)
if err != nil {
log.Printf("Failed to run command: %v", err)
}
}

17
routes/routes.go
View file

@ -56,6 +56,7 @@ func ResetGraphHandler(c *gin.Context) {
memgraph.DropDatabase(Driver) memgraph.DropDatabase(Driver)
graph := lnd.PullGraph(LndServices) graph := lnd.PullGraph(LndServices)
lnd.WriteGraphToMemgraph(graph, Driver) lnd.WriteGraphToMemgraph(graph, Driver)
memgraph.SetupAfterImport(Driver)
//IsRoutineRunning = true //IsRoutineRunning = true
StopChannel = make(chan bool) StopChannel = make(chan bool)
@ -64,6 +65,22 @@ func ResetGraphHandler(c *gin.Context) {
c.String(http.StatusOK, "Graph update started.") c.String(http.StatusOK, "Graph update started.")
} }
func LoadLocalSnapshot(c *gin.Context) {
fmt.Println("Graph update initiated...")
if IsRoutineRunning {
IsRoutineRunning = false
StopChannel <- true
close(StopChannel)
}
memgraph.DropDatabase(Driver)
lnd.WriteSnapshotToMemgraph("./describegraph.json", Driver)
memgraph.SetupAfterImport(Driver)
c.String(http.StatusOK, "Graph update started.")
}
func GetStatusHandler(c *gin.Context) { func GetStatusHandler(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"isRoutineRunning": IsRoutineRunning, c.JSON(http.StatusOK, gin.H{"isRoutineRunning": IsRoutineRunning,
"message": "Routine stopped."}) "message": "Routine stopped."})

13
static/script.js
View file

@ -72,6 +72,19 @@ function enableButtons() {
document.getElementById("toggleButton").disabled = false; document.getElementById("toggleButton").disabled = false;
} }
document.getElementById("localButton").onclick = function () {
alert("Graph reset initiated.");
disableButtons(); // Disable buttons when the request is initiated
fetch('/load-local-snapshot',{cache: "no-store"}).then(response => {
if (response.ok) {
alert("Graph reset OK.");
} else {
alert("Failed to initiate graph reset.");
}
enableButtons(); // Enable buttons when the request is completed
});
};
document.getElementById("resetButton").onclick = function () { document.getElementById("resetButton").onclick = function () {
alert("Graph reset initiated."); alert("Graph reset initiated.");
disableButtons(); // Disable buttons when the request is initiated disableButtons(); // Disable buttons when the request is initiated