pax_global_header00006660000000000000000000000064130203307050014503gustar00rootroot0000000000000052 comment=6ed3ad4f09baaf988baec174ed1f91313f7302ca Sia-1.0.4/000077500000000000000000000000001302033070500122215ustar00rootroot00000000000000Sia-1.0.4/.gitignore000066400000000000000000000002271302033070500142120ustar00rootroot00000000000000# vim *.swp *.swo # build artifacts cover release # pdflatex doc/whitepaper.aux doc/whitepaper.log doc/whitepaper.pdf # IntelliJ IDEA .idea sia.iml Sia-1.0.4/.travis.yml000066400000000000000000000003751302033070500143370ustar00rootroot00000000000000language: go os: - linux go: - 1.6 install: - make dependencies - test -z "$(go fmt ./...)" - make script: make test && make test-long && make cover && make bench sudo: false branches: only: - master notifications: email: false Sia-1.0.4/LICENSE000066400000000000000000000020701302033070500132250ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2016 Nebulous Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Sia-1.0.4/Makefile000066400000000000000000000116571302033070500136730ustar00rootroot00000000000000# all will build and install developer binaries, which have debugging enabled # and much faster mining and block constants. all: install # dependencies installs all of the dependencies that are required for building # Sia. dependencies: # Consensus Dependencies go get -u github.com/NebulousLabs/demotemutex go get -u github.com/NebulousLabs/ed25519 go get -u github.com/NebulousLabs/merkletree go get -u github.com/NebulousLabs/bolt go get -u github.com/dchest/blake2b # Module + Daemon Dependencies go get -u github.com/NebulousLabs/entropy-mnemonics go get -u github.com/NebulousLabs/go-upnp go get -u github.com/NebulousLabs/muxado go get -u github.com/klauspost/reedsolomon go get -u github.com/julienschmidt/httprouter go get -u github.com/inconshreveable/go-update go get -u github.com/kardianos/osext # Frontend Dependencies go get -u github.com/bgentry/speakeasy go get -u github.com/spf13/cobra # Developer Dependencies go install -race std go get -u github.com/golang/lint/golint # pkgs changes which packages the makefile calls operate on. run changes which # tests are run during testing. run = Test pkgs = ./api ./build ./compatibility ./crypto ./encoding ./modules ./modules/consensus \ ./modules/explorer ./modules/gateway ./modules/host ./modules/host/storagemanager \ ./modules/renter ./modules/renter/contractor ./modules/renter/hostdb ./modules/renter/proto \ ./modules/miner ./modules/wallet ./modules/transactionpool ./persist ./siac ./siad ./sync ./types # fmt calls go fmt on all packages. fmt: gofmt -s -l -w $(pkgs) # vet calls go vet on all packages. # NOTE: go vet requires packages to be built in order to obtain type info. vet: release-std go vet $(pkgs) # will always run on some packages for a while. lintpkgs = ./modules ./modules/gateway ./modules/host ./modules/renter/hostdb ./modules/renter/contractor ./persist lint: @for package in $(lintpkgs); do \ golint -min_confidence=1.0 $$package \ && test -z $$(golint -min_confidence=1.0 $$package) ; \ done # install builds and installs developer binaries. install: go install -race -tags='dev debug profile' $(pkgs) # release builds and installs release binaries. release: go install -tags='debug profile' $(pkgs) release-race: go install -race -tags='debug profile' $(pkgs) release-std: go install $(pkgs) # clean removes all directories that get automatically created during # development. clean: rm -rf release doc/whitepaper.aux doc/whitepaper.log doc/whitepaper.pdf test: go test -short -tags='debug testing' -timeout=5s $(pkgs) -run=$(run) test-v: go test -race -v -short -tags='debug testing' -timeout=15s $(pkgs) -run=$(run) test-long: clean fmt vet lint go test -v -race -tags='testing debug' -timeout=500s $(pkgs) -run=$(run) bench: clean fmt go test -tags='testing' -timeout=500s -run=XXX -bench=. $(pkgs) cover: clean @mkdir -p cover/modules @mkdir -p cover/modules/renter @mkdir -p cover/modules/host @for package in $(pkgs); do \ go test -tags='testing debug' -timeout=500s -covermode=atomic -coverprofile=cover/$$package.out ./$$package \ && go tool cover -html=cover/$$package.out -o=cover/$$package.html \ && rm cover/$$package.out ; \ done cover-integration: clean @mkdir -p cover/modules @mkdir -p cover/modules/renter @mkdir -p cover/modules/host @for package in $(pkgs); do \ go test -run=TestIntegration -tags='testing debug' -timeout=500s -covermode=atomic -coverprofile=cover/$$package.out ./$$package \ && go tool cover -html=cover/$$package.out -o=cover/$$package.html \ && rm cover/$$package.out ; \ done cover-unit: clean @mkdir -p cover/modules @mkdir -p cover/modules/renter @mkdir -p cover/modules/host @for package in $(pkgs); do \ go test -run=TestUnit -tags='testing debug' -timeout=500s -covermode=atomic -coverprofile=cover/$$package.out ./$$package \ && go tool cover -html=cover/$$package.out -o=cover/$$package.html \ && rm cover/$$package.out ; \ done # whitepaper builds the whitepaper from whitepaper.tex. pdflatex has to be # called twice because references will not update correctly the first time. whitepaper: @pdflatex -output-directory=doc whitepaper.tex > /dev/null pdflatex -output-directory=doc whitepaper.tex .PHONY: all dependencies fmt install release release-std xc clean test test-v test-long cover cover-integration cover-unit whitepaper Sia-1.0.4/README.md000066400000000000000000000161651302033070500135110ustar00rootroot00000000000000Sia 1.0.4 ========= [![Build Status](https://travis-ci.org/NebulousLabs/Sia.svg?branch=master)](https://travis-ci.org/NebulousLabs/Sia) [![GoDoc](https://godoc.org/github.com/NebulousLabs/Sia?status.svg)](https://godoc.org/github.com/NebulousLabs/Sia) [![Go Report Card](https://goreportcard.com/badge/github.com/NebulousLabs/Sia)](https://goreportcard.com/report/github.com/NebulousLabs/Sia) Binaries can be found at [our website](http://siacoin.com). Code for the graphical front-end can be found at the [Sia-UI](https://github.com/NebulousLabs/Sia-UI) repo. Sia is a new decentralized cloud storage platform aimed at giving users control of their data. Data is split into hundreds of erasure coded pieces and encrypted locally, and then each piece is uploaded to a separate host. A blockchain is used to create cryptographic contracts ensuring that hosts will only get paid if they actually store the data. Out of hundreds of hosts, only a fraction are required to recover the original file. Anybody can join the network as a host and get income from the storage they contribute. This openness allows Sia to build and take advantage of a global network of small datacenters. Combined with advanced algorithms for storing and retrieving data, Sia is poised to be a highly competitive cloud storage platform. More information about the technology can be found on our website and in the 'doc' folder of the repo. Sia is ready for use with small sums of money and non-critical files, but until the network has a more proven track record, we advise against using it as a sole means of storing important data. This release comes with 2 binaries, siad and siac. siad is a background service, or "daemon," that runs the Sia protocol, and siac is a client that is used to interact with siad. Siad exposes an HTTP API on 'localhost:9980' which can be used to interact with the daemon. There is a front-end program called Sia-UI which can be used to interact with the daemon in a more user-friendly way. Documentation on the API can be found in doc/API.md. Usage ----- siad and siac are run via command prompt. On Windows, you can just double- click siad.exe if you don't need to specify any command-line arguments. Otherwise, navigate to the sia folder and click File->Open command prompt. Then, start the siad service by entering `siad` and pressing Enter. The command prompt may appear to freeze; this means siad is waiting for requests. Windows users may see a warning from the Windows Firewall; be sure to check both boxes ("Private networks" and "Public networks") and click "Allow access." You can now run `siac` in a separate command prompt to interact with siad. From here, you can send money, mine blocks, upload and download files, and advertise yourself as a host. Building From Source -------------------- To build from source, [Go 1.6 must be installed](https://golang.org/doc/install) on the system. Then simply use `go get`: ``` go get -u github.com/NebulousLabs/Sia/... ``` This will download the Sia repo to your `$GOPATH/src` folder, and install the `siad` and `siac` binaries in your `$GOPATH/bin` folder. To stay up-to-date, run the previous `go get` command again. Alternatively, you can use the Makefile provided in this repo. Run `git pull origin master` to pull the latest changes, and `make release-std` to build the new binaries. You can also run `make test` and `make test-long` to run the short and full test suites, respectively. Finally, `make cover` will generate code coverage reports for each package; they are stored in the `cover` folder and can be viewed in your browser. Troubleshooting --------------- - I can't add storage folders on Windows. Currently, admin privileges are required to add storage folders on Windows. Close siad and run it from an admin command prompt. Future versions will not require admin privileges. - I can't connect to more than 8 peers. Once Sia has connected to 8 peers, it will stop trying to form new connections, but it will still accept incoming connection requests (up to 128 total peers). However, if you are behind a firewall, you will not be able to accept incoming connections. You must configure your firewall to allow Sia connections by forwarding your ports. By default, Sia communicates on ports 9981 and 9982. The specific instructions for forwarding a port vary by router. For more information, consult [this guide](http://portfoward.com). Sia currently has support for UPnP. While not all routers support UPnP, a majority of users should have their ports automatically forwarded by UPnP. - I mined a block, but I didn't receive any money. There is a 144-block confirmation delay before you will receive siacoins from mined blocks. If you still have not received the block reward after 144 blocks, it means your block did not make it into the blockchain. If your issue is not addressed above, you can get in touch with us personally: slack: http://slackin.siacoin.com (ping @taek, @nemo, or @jordan) email: david@nebulouslabs.com luke@nebulouslabs.com jordan@nebulouslabs.com Version Information ------------------- - v1.0.0 represents a landmark in the development of Sia. In accordance with semver, API compatibility will not be broken until v2.0.0, which is not on the current development roadmap. In other words, developers should feel confident leveraging the siad API in their own applications. When new functionality is added, it will be added in a backwards-compatible manner. New routes may be added, and new parameters may be added to existing routes or responses, but none of the routes present in v1.0.0 will be removed, nor will any of their parameters or response fields be removed. - v1.0.4 has a companion LTS release, v1.0.4-lts. Please read our [forum post](http://forum.sia.tech/topic/793/november-2016-update-roadmap) for more information. Please tell us about any problems you run into, and any features you want! The advantage of being a beta user is that your feedback will have a large impact on what we do in the next few months. Thank you! Version History --------------- December 2016: v1.0.4 (LTS release) October 2016: v1.0.3 (patch release) - Greatly improved renter stability - Smarter HostDB - Numerous minor bug fixes July 2016: v1.0.1 (patch release) - Restricted API address to localhost - Fixed renter/host desynchronization - Fixed host silently refusing new contracts June 2016: v1.0.0 (major release) - Finalized API routes - Add optional API authentication - Improve automatic contract management May 2016: v0.6.0 (minor release) - Switched to long-form renter contracts - Added support for multiple hosting folders - Hosts are now identified by their public key January 2016: v0.5.2 (patch) - Faster initial blockchain download - Introduced headers-only broadcasting v0.5.1 (patch) - Fixed bug severely impacting performance - Restored (but deprecated) some siac commands - Added modules flag, allowing modules to be disabled v0.5.0 (minor release) - Major API changes to most modules - Automatic contract renewal - Data on inactive hosts is reuploaded - Support for folder structure - Smarter host For older release notes, consult the git history of this file. Sia-1.0.4/api/000077500000000000000000000000001302033070500127725ustar00rootroot00000000000000Sia-1.0.4/api/api.go000066400000000000000000000303541302033070500140770ustar00rootroot00000000000000package api import ( "encoding/json" "net/http" "strings" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/julienschmidt/httprouter" ) // Error is a type that is encoded as JSON and returned in an API response in // the event of an error. Only the Message field is required. More fields may // be added to this struct in the future for better error reporting. type Error struct { // Message describes the error in English. Typically it is set to // `err.Error()`. This field is required. Message string `json:"message"` // TODO: add a Param field with the (omitempty option in the json tag) // to indicate that the error was caused by an invalid, missing, or // incorrect parameter. This is not trivial as the API does not // currently do parameter validation itself. For example, the // /gateway/connect endpoint relies on the gateway.Connect method to // validate the netaddress. However, this prevents the API from knowing // whether an error returned by gateway.Connect is because of a // connection error or an invalid netaddress parameter. Validating // parameters in the API is not sufficient, as a parameter's value may // be valid or invalid depending on the current state of a module. } // Error implements the error interface for the Error type. It returns only the // Message field. func (err Error) Error() string { return err.Message } // HttpGET is a utility function for making http get requests to sia with a // whitelisted user-agent. A non-2xx response does not return an error. func HttpGET(url string) (resp *http.Response, err error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", "Sia-Agent") return http.DefaultClient.Do(req) } // HttpGETAuthenticated is a utility function for making authenticated http get // requests to sia with a whitelisted user-agent and the supplied password. A // non-2xx response does not return an error. func HttpGETAuthenticated(url string, password string) (resp *http.Response, err error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", "Sia-Agent") req.SetBasicAuth("", password) return http.DefaultClient.Do(req) } // HttpPOST is a utility function for making post requests to sia with a // whitelisted user-agent. A non-2xx response does not return an error. func HttpPOST(url string, data string) (resp *http.Response, err error) { req, err := http.NewRequest("POST", url, strings.NewReader(data)) if err != nil { return nil, err } req.Header.Set("User-Agent", "Sia-Agent") req.Header.Set("Content-Type", "application/x-www-form-urlencoded") return http.DefaultClient.Do(req) } // HttpPOSTAuthenticated is a utility function for making authenticated http // post requests to sia with a whitelisted user-agent and the supplied // password. A non-2xx response does not return an error. func HttpPOSTAuthenticated(url string, data string, password string) (resp *http.Response, err error) { req, err := http.NewRequest("POST", url, strings.NewReader(data)) if err != nil { return nil, err } req.Header.Set("User-Agent", "Sia-Agent") req.Header.Set("Content-Type", "application/x-www-form-urlencoded") req.SetBasicAuth("", password) return http.DefaultClient.Do(req) } // RequireUserAgent is middleware that requires all requests to set a // UserAgent that contains the specified string. func RequireUserAgent(h http.Handler, ua string) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if !strings.Contains(req.UserAgent(), ua) { WriteError(w, Error{"Browser access disabled due to security vulnerability. Use Sia-UI or siac."}, http.StatusBadRequest) return } h.ServeHTTP(w, req) }) } // RequirePassword is middleware that requires a request to authenticate with a // password using HTTP basic auth. Usernames are ignored. Empty passwords // indicate no authentication is required. func RequirePassword(h httprouter.Handle, password string) httprouter.Handle { // An empty password is equivalent to no password. if password == "" { return h } return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { _, pass, ok := req.BasicAuth() if !ok || pass != password { w.Header().Set("WWW-Authenticate", "Basic realm=\"SiaAPI\"") WriteError(w, Error{"API authentication failed."}, http.StatusUnauthorized) return } h(w, req, ps) } } // API encapsulates a collection of modules and implements a http.Handler // to access their methods. type API struct { cs modules.ConsensusSet explorer modules.Explorer gateway modules.Gateway host modules.Host miner modules.Miner renter modules.Renter tpool modules.TransactionPool wallet modules.Wallet router http.Handler } // api.ServeHTTP implements the http.Handler interface. func (api *API) ServeHTTP(w http.ResponseWriter, r *http.Request) { api.router.ServeHTTP(w, r) } // New creates a new Sia API from the provided modules. The API will require // authentication using HTTP basic auth for certain endpoints of the supplied // password is not the empty string. Usernames are ignored for authentication. func New(requiredUserAgent string, requiredPassword string, cs modules.ConsensusSet, e modules.Explorer, g modules.Gateway, h modules.Host, m modules.Miner, r modules.Renter, tp modules.TransactionPool, w modules.Wallet) *API { api := &API{ cs: cs, explorer: e, gateway: g, host: h, miner: m, renter: r, tpool: tp, wallet: w, } // Register API handlers router := httprouter.New() router.NotFound = http.HandlerFunc(UnrecognizedCallHandler) // Consensus API Calls if api.cs != nil { router.GET("/consensus", api.consensusHandler) } // Explorer API Calls if api.explorer != nil { router.GET("/explorer", api.explorerHandler) router.GET("/explorer/blocks/:height", api.explorerBlocksHandler) router.GET("/explorer/hashes/:hash", api.explorerHashHandler) } // Gateway API Calls if api.gateway != nil { router.GET("/gateway", api.gatewayHandler) router.POST("/gateway/connect/:netaddress", RequirePassword(api.gatewayConnectHandler, requiredPassword)) router.POST("/gateway/disconnect/:netaddress", RequirePassword(api.gatewayDisconnectHandler, requiredPassword)) } // Host API Calls if api.host != nil { // Calls directly pertaining to the host. router.GET("/host", api.hostHandlerGET) // Get the host status. router.POST("/host", RequirePassword(api.hostHandlerPOST, requiredPassword)) // Change the settings of the host. router.POST("/host/announce", RequirePassword(api.hostAnnounceHandler, requiredPassword)) // Announce the host to the network. // Calls pertaining to the storage manager that the host uses. router.GET("/host/storage", api.storageHandler) router.POST("/host/storage/folders/add", RequirePassword(api.storageFoldersAddHandler, requiredPassword)) router.POST("/host/storage/folders/remove", RequirePassword(api.storageFoldersRemoveHandler, requiredPassword)) router.POST("/host/storage/folders/resize", RequirePassword(api.storageFoldersResizeHandler, requiredPassword)) router.POST("/host/storage/sectors/delete/:merkleroot", RequirePassword(api.storageSectorsDeleteHandler, requiredPassword)) } // Miner API Calls if api.miner != nil { router.GET("/miner", api.minerHandler) router.GET("/miner/header", RequirePassword(api.minerHeaderHandlerGET, requiredPassword)) router.POST("/miner/header", RequirePassword(api.minerHeaderHandlerPOST, requiredPassword)) router.GET("/miner/start", RequirePassword(api.minerStartHandler, requiredPassword)) router.GET("/miner/stop", RequirePassword(api.minerStopHandler, requiredPassword)) } // Renter API Calls if api.renter != nil { router.GET("/renter", api.renterHandlerGET) router.POST("/renter", RequirePassword(api.renterHandlerPOST, requiredPassword)) router.GET("/renter/contracts", api.renterContractsHandler) router.GET("/renter/downloads", api.renterDownloadsHandler) router.GET("/renter/files", api.renterFilesHandler) // TODO: re-enable these routes once the new .sia format has been // standardized and implemented. // router.POST("/renter/load", RequirePassword(api.renterLoadHandler, requiredPassword)) // router.POST("/renter/loadascii", RequirePassword(api.renterLoadAsciiHandler, requiredPassword)) // router.GET("/renter/share", RequirePassword(api.renterShareHandler, requiredPassword)) // router.GET("/renter/shareascii", RequirePassword(api.renterShareAsciiHandler, requiredPassword)) router.POST("/renter/delete/*siapath", RequirePassword(api.renterDeleteHandler, requiredPassword)) router.GET("/renter/download/*siapath", RequirePassword(api.renterDownloadHandler, requiredPassword)) router.POST("/renter/rename/*siapath", RequirePassword(api.renterRenameHandler, requiredPassword)) router.POST("/renter/upload/*siapath", RequirePassword(api.renterUploadHandler, requiredPassword)) // HostDB endpoints. router.GET("/hostdb/active", api.renterHostsActiveHandler) router.GET("/hostdb/all", api.renterHostsAllHandler) } // TransactionPool API Calls if api.tpool != nil { // TODO: re-enable this route once the transaction pool API has been finalized //router.GET("/transactionpool/transactions", api.transactionpoolTransactionsHandler) } // Wallet API Calls if api.wallet != nil { router.GET("/wallet", api.walletHandler) router.POST("/wallet/033x", RequirePassword(api.wallet033xHandler, requiredPassword)) router.GET("/wallet/address", RequirePassword(api.walletAddressHandler, requiredPassword)) router.GET("/wallet/addresses", api.walletAddressesHandler) router.GET("/wallet/backup", RequirePassword(api.walletBackupHandler, requiredPassword)) router.POST("/wallet/init", RequirePassword(api.walletInitHandler, requiredPassword)) router.POST("/wallet/lock", RequirePassword(api.walletLockHandler, requiredPassword)) router.POST("/wallet/seed", RequirePassword(api.walletSeedHandler, requiredPassword)) router.GET("/wallet/seeds", RequirePassword(api.walletSeedsHandler, requiredPassword)) router.POST("/wallet/siacoins", RequirePassword(api.walletSiacoinsHandler, requiredPassword)) router.POST("/wallet/siafunds", RequirePassword(api.walletSiafundsHandler, requiredPassword)) router.POST("/wallet/siagkey", RequirePassword(api.walletSiagkeyHandler, requiredPassword)) router.GET("/wallet/transaction/:id", api.walletTransactionHandler) router.GET("/wallet/transactions", api.walletTransactionsHandler) router.GET("/wallet/transactions/:addr", api.walletTransactionsAddrHandler) router.POST("/wallet/unlock", RequirePassword(api.walletUnlockHandler, requiredPassword)) } // Apply UserAgent middleware and return the API api.router = RequireUserAgent(router, requiredUserAgent) return api } // UnrecognizedCallHandler handles calls to unknown pages (404). func UnrecognizedCallHandler(w http.ResponseWriter, req *http.Request) { WriteError(w, Error{"404 - Refer to API.md"}, http.StatusNotFound) } // WriteError an error to the API caller. func WriteError(w http.ResponseWriter, err Error, code int) { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(code) if encodeErr := json.NewEncoder(w).Encode(err); encodeErr != nil { // Marshalling should only fail in the event of a developer error. // Specifically, only non-marshallable types should cause an error here. build.Critical("failed to encode API error response:", encodeErr) } } // WriteJSON writes the object to the ResponseWriter. If the encoding fails, an // error is written instead. The Content-Type of the response header is set // accordingly. func WriteJSON(w http.ResponseWriter, obj interface{}) { w.Header().Set("Content-Type", "application/json; charset=utf-8") if err := json.NewEncoder(w).Encode(obj); err != nil { // Marshalling should only fail in the event of a developer error. // Specifically, only non-marshallable types should cause an error here. build.Critical("failed to encode API response:", err) } } // WriteSuccess writes the HTTP header with status 204 No Content to the // ResponseWriter. WriteSuccess should only be used to indicate that the // requested action succeeded AND there is no data to return. func WriteSuccess(w http.ResponseWriter) { w.WriteHeader(http.StatusNoContent) } Sia-1.0.4/api/client.go000066400000000000000000000057071302033070500146100ustar00rootroot00000000000000package api import ( "encoding/json" "errors" "io" "io/ioutil" "net/http" "strings" ) // Client holds fields to make requests to a Sia API. type Client struct { address string password string } // NewClient creates a new api.Client using the provided address and password. // If password is not the empty string, HTTP basic authentication will be used // to communicate with the API. func NewClient(address string, password string) *Client { return &Client{ address: address, password: password, } } // Get requests the resource at `resource` and decodes it into `obj`, returning an // error if requesting or decoding the resource fails. A non-2xx status code // constitutes a request failure. func (c *Client) Get(resource string, obj interface{}) error { url := "http://" + c.address + resource req, err := http.NewRequest("GET", url, nil) if err != nil { return err } req.Header.Set("User-Agent", "Sia-Agent") if c.password != "" { req.SetBasicAuth("", c.password) } res, err := http.DefaultClient.Do(req) if err != nil { return err } defer func() { // res.Body should always be fully read even when discarding its content, // such that the underlying connection can be reused. io.Copy(ioutil.Discard, res.Body) res.Body.Close() }() if res.StatusCode == http.StatusNotFound { return errors.New("API call not recognized: " + resource) } // Decode the body as an Error and return this error if the status code is // not 2xx. if res.StatusCode < 200 || res.StatusCode > 299 { var apiErr Error err = json.NewDecoder(res.Body).Decode(&apiErr) if err != nil { return err } return apiErr } if res.StatusCode != http.StatusNoContent && obj != nil { return json.NewDecoder(res.Body).Decode(obj) } return nil } // Post makes a POST request to the resource at `resource`, using `data` as the // request body. The response, if provided, will be decoded into `obj`. func (c *Client) Post(resource string, data string, obj interface{}) error { url := "http://" + c.address + resource req, err := http.NewRequest("POST", url, strings.NewReader(data)) if err != nil { return err } req.Header.Set("User-Agent", "Sia-Agent") req.Header.Set("Content-Type", "application/x-www-form-urlencoded") if c.password != "" { req.SetBasicAuth("", c.password) } res, err := http.DefaultClient.Do(req) if err != nil { return err } defer func() { // res.Body should always be fully read even when discarding its content, // such that the underlying connection can be reused. io.Copy(ioutil.Discard, res.Body) res.Body.Close() }() if res.StatusCode == http.StatusNotFound { return errors.New("API call not recognized: " + resource) } if res.StatusCode < 200 || res.StatusCode > 299 { var apiErr Error err = json.NewDecoder(res.Body).Decode(&apiErr) if err != nil { return err } return apiErr } if res.StatusCode != http.StatusNoContent && obj != nil { return json.NewDecoder(res.Body).Decode(&obj) } return nil } Sia-1.0.4/api/client_test.go000066400000000000000000000025321302033070500156400ustar00rootroot00000000000000package api import ( "testing" ) // TestApiClient tests that the API client connects to the server tester and // can call and decode routes correctly. func TestIntegrationApiClient(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestApiClient") if err != nil { t.Fatal(err) } defer st.server.Close() c := NewClient(st.server.listener.Addr().String(), "") var gatewayInfo GatewayGET err = c.Get("/gateway", &gatewayInfo) if err != nil { t.Fatal(err) } } // TestAuthenticatedApiClient tests that the API client connects to an // authenticated server tester and can call and decode routes correctly, using // the correct password. func TestIntegrationAuthenticatedApiClient(t *testing.T) { if testing.Short() { t.SkipNow() } testpass := "testPassword" st, err := createAuthenticatedServerTester("TestAuthenticatedApiClient", testpass) if err != nil { t.Fatal(err) } defer st.server.Close() c := NewClient(st.server.listener.Addr().String(), "") var walletAddress WalletAddressGET err = c.Get("/wallet/address", &walletAddress) if err == nil { t.Fatal("api.Client did not return an error when requesting an authenticated resource without a password") } c = NewClient(st.server.listener.Addr().String(), testpass) err = c.Get("/wallet/address", &walletAddress) if err != nil { t.Fatal(err) } } Sia-1.0.4/api/consensus.go000066400000000000000000000015251302033070500153440ustar00rootroot00000000000000package api import ( "net/http" "github.com/NebulousLabs/Sia/types" "github.com/julienschmidt/httprouter" ) // ConsensusGET contains general information about the consensus set, with tags // to support idiomatic json encodings. type ConsensusGET struct { Synced bool `json:"synced"` Height types.BlockHeight `json:"height"` CurrentBlock types.BlockID `json:"currentblock"` Target types.Target `json:"target"` } // consensusHandler handles the API calls to /consensus. func (api *API) consensusHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { cbid := api.cs.CurrentBlock().ID() currentTarget, _ := api.cs.ChildTarget(cbid) WriteJSON(w, ConsensusGET{ Synced: api.cs.Synced(), Height: api.cs.Height(), CurrentBlock: cbid, Target: currentTarget, }) } Sia-1.0.4/api/consensus_test.go000066400000000000000000000014121302033070500163760ustar00rootroot00000000000000package api import ( "testing" "github.com/NebulousLabs/Sia/types" ) // TestIntegrationConsensusGet probes the GET call to /consensus. func TestIntegrationConsensusGET(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestIntegrationConsensusGET") if err != nil { t.Fatal(err) } defer st.server.Close() var cg ConsensusGET err = st.getAPI("/consensus", &cg) if err != nil { t.Fatal(err) } if cg.Height != 4 { t.Error("wrong height returned in consensus GET call") } if cg.CurrentBlock != st.server.api.cs.CurrentBlock().ID() { t.Error("wrong block returned in consensus GET call") } expectedTarget := types.Target{128} if cg.Target != expectedTarget { t.Error("wrong target returned in consensus GET call") } } Sia-1.0.4/api/ecosystem_test.go000066400000000000000000000153021302033070500163740ustar00rootroot00000000000000package api // ecosystem_test.go provides tooling and tests for whole-ecosystem testing, // consisting of multiple full, non-state-sharing nodes connected in various // arrangements and performing various full-ecosystem tasks. import ( "errors" "testing" "time" "github.com/NebulousLabs/Sia/types" ) // synchronizationCheck takes a bunch of server testers as input and checks // that they all have the same current block as the first server tester. The // first server tester needs to have the most recent block in order for the // check to work. func synchronizationCheck(sts []*serverTester) (types.BlockID, error) { // Prefer returning an error in the event of a zero-length server tester - // an error should be returned if the developer accidentally uses a nil // slice instead of whatever value was intended, and there's no reason to // check for synchronization if there aren't any nodes to be synchronized. if len(sts) == 0 { return types.BlockID{}, errors.New("no server testers provided") } var cg ConsensusGET err := sts[0].getAPI("/consensus", &cg) if err != nil { return types.BlockID{}, err } leaderBlockID := cg.CurrentBlock for i := range sts { // Spin until the current block matches the leader block. success := false for j := 0; j < 100; j++ { err = sts[i].getAPI("/consensus", &cg) if err != nil { return types.BlockID{}, err } if cg.CurrentBlock == leaderBlockID { success = true break } time.Sleep(time.Millisecond * 100) } if !success { return types.BlockID{}, errors.New("synchronization check failed - nodes do not seem to be synchronized") } } return leaderBlockID, nil } // TestHostPoorConnectivity creates several full server testers and links them // together in a way that might mimic a full host ecosystem with a renter, and // then isolates one of the hosts from the network, denying the host proper // transaction propagation. The renters performed chained contract forming and // uploading in the same manner that might happen in the wild, and then the // host must get a file contract to the blockchain despite not getting any of // the dependencies into the transaction pool from the flood network. func TestHostPoorConnectivity(t *testing.T) { if testing.Short() { t.SkipNow() } // Create the various nodes that will be forming the simulated ecosystem of // this test. stLeader, err := createServerTester("TestHostPoorConnectivity - Leader") if err != nil { t.Fatal(err) } stHost1, err := blankServerTester("TestHostPoorConnectivity - Host 1") if err != nil { t.Fatal(err) } stHost2, err := blankServerTester("TestHostPoorConnectivity - Host 2") if err != nil { t.Fatal(err) } stHost3, err := blankServerTester("TestHostPoorConnectivity - Host 3") if err != nil { t.Fatal(err) } stHost4, err := blankServerTester("TestHostPoorConnectivity - Host 4") if err != nil { t.Fatal(err) } stRenter1, err := blankServerTester("TestHostPoorConnectivity - Renter 1") if err != nil { t.Fatal(err) } stRenter2, err := blankServerTester("TestHostPoorConnectivity - Renter 2") if err != nil { t.Fatal(err) } // Fetch all of the addresses of the nodes that got created. var ggSTL, ggSTH1, ggSTH2, ggSTH3, ggSTH4, ggSTR1, ggSTR2 GatewayGET err = stLeader.getAPI("/gateway", &ggSTL) if err != nil { t.Fatal(err) } err = stHost1.getAPI("/gateway", &ggSTH1) if err != nil { t.Fatal(err) } err = stHost2.getAPI("/gateway", &ggSTH2) if err != nil { t.Fatal(err) } err = stHost3.getAPI("/gateway", &ggSTH3) if err != nil { t.Fatal(err) } err = stHost4.getAPI("/gateway", &ggSTH4) if err != nil { t.Fatal(err) } err = stRenter1.getAPI("/gateway", &ggSTR1) if err != nil { t.Fatal(err) } err = stRenter2.getAPI("/gateway", &ggSTR2) if err != nil { t.Fatal(err) } // Connect all of the peers in a circle, so that everyone is connected but // there are a lot of hops. err = stLeader.stdPostAPI("/gateway/connect/"+string(ggSTH1.NetAddress), nil) if err != nil { t.Fatal(err) } err = stHost1.stdPostAPI("/gateway/connect/"+string(ggSTH2.NetAddress), nil) if err != nil { t.Fatal(err) } err = stHost2.stdPostAPI("/gateway/connect/"+string(ggSTH3.NetAddress), nil) if err != nil { t.Fatal(err) } err = stHost3.stdPostAPI("/gateway/connect/"+string(ggSTH4.NetAddress), nil) if err != nil { t.Fatal(err) } err = stHost4.stdPostAPI("/gateway/connect/"+string(ggSTR1.NetAddress), nil) if err != nil { t.Fatal(err) } err = stRenter1.stdPostAPI("/gateway/connect/"+string(ggSTR2.NetAddress), nil) if err != nil { t.Fatal(err) } err = stRenter2.stdPostAPI("/gateway/connect/"+string(ggSTL.NetAddress), nil) if err != nil { t.Fatal(err) } // Connectivity check - all nodes should be synchronized to the leader's // chain, which should have been the longest. allTesters := []*serverTester{stLeader, stHost1, stHost2, stHost3, stHost4, stRenter1, stRenter2} chainTip, err := synchronizationCheck(allTesters) if err != nil { t.Fatal(err) } // Mine a block from each node, to give the node money in the wallet that // is recognized by the shared chain. for i := range allTesters { // Wait until the current tester has 'chainTip' as its current // block, to make sure the network is building a community chain // instead of creating orphans. var cg ConsensusGET success := false for j := 0; j < 100; j++ { err = allTesters[i].getAPI("/consensus", &cg) if err != nil { t.Fatal(err) } if cg.CurrentBlock == chainTip { success = true break } time.Sleep(time.Millisecond * 100) } if !success { t.Fatal("nodes do not seem to be synchronizing") } err := allTesters[i].cs.Flush() if err != nil { t.Fatal(err) } // Mine a block for this node. The next iteration will wait for // synchronization before mining the block for the next node. block, err := allTesters[i].miner.AddBlock() if err != nil { t.Fatal(err, i) } chainTip = block.ID() } // Wait until the leader has the most recent block. var cg ConsensusGET success := false for i := 0; i < 100; i++ { err = allTesters[0].getAPI("/consensus", &cg) if err != nil { t.Fatal(err) } if cg.CurrentBlock == chainTip { success = true break } time.Sleep(time.Millisecond * 100) } if !success { t.Fatal("nodes do not seem to be synchronizing") } // Make sure that everyone has the most recent block. _, err = synchronizationCheck(allTesters) if err != nil { t.Fatal(err) } // Mine blocks from the leader until everyone's miner payouts have matured // and become spendable. for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ { _, err := stLeader.miner.AddBlock() if err != nil { t.Fatal(err) } } _, err = synchronizationCheck(allTesters) if err != nil { t.Fatal(err) } } Sia-1.0.4/api/explorer.go000066400000000000000000000326571302033070500151760ustar00rootroot00000000000000package api import ( "fmt" "net/http" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/julienschmidt/httprouter" ) type ( // ExplorerBlock is a block with some extra information such as the id and // height. This information is provided for programs that may not be // complex enough to compute the ID on their own. ExplorerBlock struct { MinerPayoutIDs []types.SiacoinOutputID `json:"minerpayoutids"` Transactions []ExplorerTransaction `json:"transactions"` RawBlock types.Block `json:"rawblock"` modules.BlockFacts } // ExplorerTransaction is a transcation with some extra information such as // the parent block. This information is provided for programs that may not // be complex enough to compute the extra information on their own. ExplorerTransaction struct { ID types.TransactionID `json:"id"` Height types.BlockHeight `json:"height"` Parent types.BlockID `json:"parent"` RawTransaction types.Transaction `json:"rawtransaction"` SiacoinInputOutputs []types.SiacoinOutput `json:"siacoininputoutputs"` // the outputs being spent SiacoinOutputIDs []types.SiacoinOutputID `json:"siacoinoutputids"` FileContractIDs []types.FileContractID `json:"filecontractids"` FileContractValidProofOutputIDs [][]types.SiacoinOutputID `json:"filecontractvalidproofoutputids"` // outer array is per-contract FileContractMissedProofOutputIDs [][]types.SiacoinOutputID `json:"filecontractmissedproofoutputids"` // outer array is per-contract FileContractRevisionValidProofOutputIDs [][]types.SiacoinOutputID `json:"filecontractrevisionvalidproofoutputids"` // outer array is per-revision FileContractRevisionMissedProofOutputIDs [][]types.SiacoinOutputID `json:"filecontractrevisionmissedproofoutputids"` // outer array is per-revision StorageProofOutputIDs [][]types.SiacoinOutputID `json:"storageproofoutputids"` // outer array is per-payout StorageProofOutputs [][]types.SiacoinOutput `json:"storageproofoutputs"` // outer array is per-payout SiafundInputOutputs []types.SiafundOutput `json:"siafundinputoutputs"` // the outputs being spent SiafundOutputIDs []types.SiafundOutputID `json:"siafundoutputids"` SiafundClaimOutputIDs []types.SiacoinOutputID `json:"siafundclaimoutputids"` } // ExplorerGET is the object returned as a response to a GET request to // /explorer. ExplorerGET struct { modules.BlockFacts } // ExplorerBlockGET is the object returned by a GET request to // /explorer/block. ExplorerBlockGET struct { Block ExplorerBlock `json:"block"` } // ExplorerHashGET is the object returned as a response to a GET request to // /explorer/hash. The HashType will indicate whether the hash corresponds // to a block id, a transaction id, a siacoin output id, a file contract // id, or a siafund output id. In the case of a block id, 'Block' will be // filled out and all the rest of the fields will be blank. In the case of // a transaction id, 'Transaction' will be filled out and all the rest of // the fields will be blank. For everything else, 'Transactions' and // 'Blocks' will/may be filled out and everything else will be blank. ExplorerHashGET struct { HashType string `json:"hashtype"` Block ExplorerBlock `json:"block"` Blocks []ExplorerBlock `json:"blocks"` Transaction ExplorerTransaction `json:"transaction"` Transactions []ExplorerTransaction `json:"transactions"` } ) // buildExplorerTransaction takes a transaction and the height + id of the // block it appears in an uses that to build an explorer transaction. func (api *API) buildExplorerTransaction(height types.BlockHeight, parent types.BlockID, txn types.Transaction) (et ExplorerTransaction) { // Get the header information for the transaction. et.ID = txn.ID() et.Height = height et.Parent = parent et.RawTransaction = txn // Add the siacoin outputs that correspond with each siacoin input. for _, sci := range txn.SiacoinInputs { sco, exists := api.explorer.SiacoinOutput(sci.ParentID) if build.DEBUG && !exists { panic("could not find corresponding siacoin output") } et.SiacoinInputOutputs = append(et.SiacoinInputOutputs, sco) } for i := range txn.SiacoinOutputs { et.SiacoinOutputIDs = append(et.SiacoinOutputIDs, txn.SiacoinOutputID(uint64(i))) } // Add all of the valid and missed proof ids as extra data to the file // contracts. for i, fc := range txn.FileContracts { fcid := txn.FileContractID(uint64(i)) var fcvpoids []types.SiacoinOutputID var fcmpoids []types.SiacoinOutputID for j := range fc.ValidProofOutputs { fcvpoids = append(fcvpoids, fcid.StorageProofOutputID(types.ProofValid, uint64(j))) } for j := range fc.MissedProofOutputs { fcmpoids = append(fcmpoids, fcid.StorageProofOutputID(types.ProofMissed, uint64(j))) } et.FileContractIDs = append(et.FileContractIDs, fcid) et.FileContractValidProofOutputIDs = append(et.FileContractValidProofOutputIDs, fcvpoids) et.FileContractMissedProofOutputIDs = append(et.FileContractMissedProofOutputIDs, fcmpoids) } // Add all of the valid and missed proof ids as extra data to the file // contract revisions. for _, fcr := range txn.FileContractRevisions { var fcrvpoids []types.SiacoinOutputID var fcrmpoids []types.SiacoinOutputID for j := range fcr.NewValidProofOutputs { fcrvpoids = append(fcrvpoids, fcr.ParentID.StorageProofOutputID(types.ProofValid, uint64(j))) } for j := range fcr.NewMissedProofOutputs { fcrmpoids = append(fcrmpoids, fcr.ParentID.StorageProofOutputID(types.ProofMissed, uint64(j))) } et.FileContractValidProofOutputIDs = append(et.FileContractValidProofOutputIDs, fcrvpoids) et.FileContractMissedProofOutputIDs = append(et.FileContractMissedProofOutputIDs, fcrmpoids) } // Add all of the output ids and outputs corresponding with each storage // proof. for _, sp := range txn.StorageProofs { fileContract, fileContractRevisions, fileContractExists, _ := api.explorer.FileContractHistory(sp.ParentID) if !fileContractExists && build.DEBUG { panic("could not find a file contract connected with a storage proof") } var storageProofOutputs []types.SiacoinOutput if len(fileContractRevisions) > 0 { storageProofOutputs = fileContractRevisions[len(fileContractRevisions)-1].NewValidProofOutputs } else { storageProofOutputs = fileContract.ValidProofOutputs } var storageProofOutputIDs []types.SiacoinOutputID for i := range storageProofOutputs { storageProofOutputIDs = append(storageProofOutputIDs, sp.ParentID.StorageProofOutputID(types.ProofValid, uint64(i))) } et.StorageProofOutputIDs = append(et.StorageProofOutputIDs, storageProofOutputIDs) et.StorageProofOutputs = append(et.StorageProofOutputs, storageProofOutputs) } // Add the siafund outputs that correspond to each siacoin input. for _, sci := range txn.SiafundInputs { sco, exists := api.explorer.SiafundOutput(sci.ParentID) if build.DEBUG && !exists { panic("could not find corresponding siafund output") } et.SiafundInputOutputs = append(et.SiafundInputOutputs, sco) } for i := range txn.SiafundOutputs { et.SiafundOutputIDs = append(et.SiafundOutputIDs, txn.SiafundOutputID(uint64(i))) } for _, sfi := range txn.SiafundInputs { et.SiafundClaimOutputIDs = append(et.SiafundClaimOutputIDs, sfi.ParentID.SiaClaimOutputID()) } return et } // buildExplorerBlock takes a block and its height and uses it to construct an // explorer block. func (api *API) buildExplorerBlock(height types.BlockHeight, block types.Block) ExplorerBlock { var mpoids []types.SiacoinOutputID for i := range block.MinerPayouts { mpoids = append(mpoids, block.MinerPayoutID(uint64(i))) } var etxns []ExplorerTransaction for _, txn := range block.Transactions { etxns = append(etxns, api.buildExplorerTransaction(height, block.ID(), txn)) } facts, exists := api.explorer.BlockFacts(height) if build.DEBUG && !exists { panic("incorrect request to buildExplorerBlock - block does not exist") } return ExplorerBlock{ MinerPayoutIDs: mpoids, Transactions: etxns, RawBlock: block, BlockFacts: facts, } } // explorerHandler handles API calls to /explorer/blocks/:height. func (api *API) explorerBlocksHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // Parse the height that's being requested. var height types.BlockHeight _, err := fmt.Sscan(ps.ByName("height"), &height) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } // Fetch and return the explorer block. block, exists := api.cs.BlockAtHeight(height) if !exists { WriteError(w, Error{"no block found at input height in call to /explorer/block"}, http.StatusBadRequest) return } WriteJSON(w, ExplorerBlockGET{ Block: api.buildExplorerBlock(height, block), }) } // buildTransactionSet returns the blocks and transactions that are associated // with a set of transaction ids. func (api *API) buildTransactionSet(txids []types.TransactionID) (txns []ExplorerTransaction, blocks []ExplorerBlock) { for _, txid := range txids { // Get the block containing the transaction - in the case of miner // payouts, the block might be the transaction. block, height, exists := api.explorer.Transaction(txid) if !exists && build.DEBUG { panic("explorer pointing to nonexistent txn") } // Check if the block is the transaction. if types.TransactionID(block.ID()) == txid { blocks = append(blocks, api.buildExplorerBlock(height, block)) } else { // Find the transaction within the block with the correct id. for _, t := range block.Transactions { if t.ID() == txid { txns = append(txns, api.buildExplorerTransaction(height, block.ID(), t)) break } } } } return txns, blocks } // explorerHashHandler handles GET requests to /explorer/hash/:hash. func (api *API) explorerHashHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // Scan the hash as a hash. If that fails, try scanning the hash as an // address. hash, err := scanHash(ps.ByName("hash")) if err != nil { addr, err := scanAddress(ps.ByName("hash")) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } hash = crypto.Hash(addr) } // TODO: lookups on the zero hash are too expensive to allow. Need a // better way to handle this case. if hash == (crypto.Hash{}) { WriteError(w, Error{"can't lookup the empty unlock hash"}, http.StatusBadRequest) return } // Try the hash as a block id. block, height, exists := api.explorer.Block(types.BlockID(hash)) if exists { WriteJSON(w, ExplorerHashGET{ HashType: "blockid", Block: api.buildExplorerBlock(height, block), }) return } // Try the hash as a transaction id. block, height, exists = api.explorer.Transaction(types.TransactionID(hash)) if exists { var txn types.Transaction for _, t := range block.Transactions { if t.ID() == types.TransactionID(hash) { txn = t } } WriteJSON(w, ExplorerHashGET{ HashType: "transactionid", Transaction: api.buildExplorerTransaction(height, block.ID(), txn), }) return } // Try the hash as a siacoin output id. txids := api.explorer.SiacoinOutputID(types.SiacoinOutputID(hash)) if len(txids) != 0 { txns, blocks := api.buildTransactionSet(txids) WriteJSON(w, ExplorerHashGET{ HashType: "siacoinoutputid", Blocks: blocks, Transactions: txns, }) return } // Try the hash as a file contract id. txids = api.explorer.FileContractID(types.FileContractID(hash)) if len(txids) != 0 { txns, blocks := api.buildTransactionSet(txids) WriteJSON(w, ExplorerHashGET{ HashType: "filecontractid", Blocks: blocks, Transactions: txns, }) return } // Try the hash as a siafund output id. txids = api.explorer.SiafundOutputID(types.SiafundOutputID(hash)) if len(txids) != 0 { txns, blocks := api.buildTransactionSet(txids) WriteJSON(w, ExplorerHashGET{ HashType: "siafundoutputid", Blocks: blocks, Transactions: txns, }) return } // Try the hash as an unlock hash. Unlock hash is checked last because // unlock hashes do not have collision-free guarantees. Someone can create // an unlock hash that collides with another object id. They will not be // able to use the unlock hash, but they can disrupt the explorer. This is // handled by checking the unlock hash last. Anyone intentionally creating // a colliding unlock hash (such a collision can only happen if done // intentionally) will be unable to find their unlock hash in the // blockchain through the explorer hash lookup. txids = api.explorer.UnlockHash(types.UnlockHash(hash)) if len(txids) != 0 { txns, blocks := api.buildTransactionSet(txids) WriteJSON(w, ExplorerHashGET{ HashType: "unlockhash", Blocks: blocks, Transactions: txns, }) return } // Hash not found, return an error. WriteError(w, Error{"unrecognized hash used as input to /explorer/hash"}, http.StatusBadRequest) } // explorerHandler handles API calls to /explorer func (api *API) explorerHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { facts := api.explorer.LatestBlockFacts() WriteJSON(w, ExplorerGET{ BlockFacts: facts, }) } Sia-1.0.4/api/explorer_test.go000066400000000000000000000037201302033070500162220ustar00rootroot00000000000000package api import ( "testing" "github.com/NebulousLabs/Sia/types" ) // TestIntegrationExplorerGET probes the GET call to /explorer. func TestIntegrationExplorerGET(t *testing.T) { t.Skip("Explorer has deadlock issues") if testing.Short() { t.SkipNow() } st, err := createServerTester("TestIntegrationExplorerGET") if err != nil { t.Fatal(err) } defer st.server.Close() var eg ExplorerGET err = st.getAPI("/explorer", &eg) if err != nil { t.Fatal(err) } if eg.Height != st.server.api.cs.Height() { t.Error("height not accurately reported by explorer") } if eg.MinerPayoutCount == 0 { t.Error("Miner payout count is incorrect") } } // TestIntegrationExplorerBlockGET probes the GET call to /explorer/block. func TestIntegrationExplorerBlockGET(t *testing.T) { t.Skip("Explorer has deadlock issues") if testing.Short() { t.SkipNow() } st, err := createServerTester("TestIntegrationExplorerBlockGET") if err != nil { t.Fatal(err) } defer st.server.Close() var ebg ExplorerBlockGET err = st.getAPI("/explorer/blocks/0", &ebg) if err != nil { t.Fatal(err) } if ebg.Block.BlockID != ebg.Block.RawBlock.ID() { t.Error("block id and block do not match up from api call") } if ebg.Block.BlockID != types.GenesisBlock.ID() { t.Error("wrong block returned by /explorer/block?height=0") } } // TestIntegrationExplorerHashGet probes the GET call to /explorer/hash/:hash. func TestIntegrationExplorerHashGet(t *testing.T) { t.Skip("Explorer has deadlock issues") if testing.Short() { t.SkipNow() } st, err := createServerTester("TestIntegrationExplorerHashGET") if err != nil { t.Fatal(err) } defer st.server.Close() var ehg ExplorerHashGET gb := types.GenesisBlock err = st.getAPI("/explorer/hashes/"+gb.ID().String(), &ehg) if err != nil { t.Fatal(err) } if ehg.HashType != "blockid" { t.Error("wrong hash type returned when requesting block hash") } if ehg.Block.BlockID != gb.ID() { t.Error("wrong block type returned") } } Sia-1.0.4/api/gateway.go000066400000000000000000000030521302033070500147620ustar00rootroot00000000000000package api import ( "net/http" "github.com/NebulousLabs/Sia/modules" "github.com/julienschmidt/httprouter" ) // GatewayGET contains the fields returned by a GET call to "/gateway". type GatewayGET struct { NetAddress modules.NetAddress `json:"netaddress"` Peers []modules.Peer `json:"peers"` } // gatewayHandler handles the API call asking for the gatway status. func (api *API) gatewayHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { peers := api.gateway.Peers() // nil slices are marshalled as 'null' in JSON, whereas 0-length slices are // marshalled as '[]'. The latter is preferred, indicating that the value // exists but contains no elements. if peers == nil { peers = make([]modules.Peer, 0) } WriteJSON(w, GatewayGET{api.gateway.Address(), peers}) } // gatewayConnectHandler handles the API call to add a peer to the gateway. func (api *API) gatewayConnectHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { addr := modules.NetAddress(ps.ByName("netaddress")) err := api.gateway.Connect(addr) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } // gatewayDisconnectHandler handles the API call to remove a peer from the gateway. func (api *API) gatewayDisconnectHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { addr := modules.NetAddress(ps.ByName("netaddress")) err := api.gateway.Disconnect(addr) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } Sia-1.0.4/api/gateway_test.go000066400000000000000000000045371302033070500160320ustar00rootroot00000000000000package api import ( "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules/gateway" ) // TestGatewayStatus checks that the /gateway/status call is returning a corect // peerlist. func TestGatewayStatus(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestGatewayStatus") if err != nil { t.Fatal(err) } defer st.server.Close() var info GatewayGET st.getAPI("/gateway", &info) if len(info.Peers) != 0 { t.Fatal("/gateway gave bad peer list:", info.Peers) } } // TestGatewayPeerConnect checks that /gateway/connect is adding a peer to the // gateway's peerlist. func TestGatewayPeerConnect(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestGatewayPeerConnect1") if err != nil { t.Fatal(err) } defer st.server.Close() peer, err := gateway.New("localhost:0", false, build.TempDir("api", "TestGatewayPeerConnect2", "gateway")) if err != nil { t.Fatal(err) } err = st.stdPostAPI("/gateway/connect/"+string(peer.Address()), nil) if err != nil { t.Fatal(err) } var info GatewayGET err = st.getAPI("/gateway", &info) if err != nil { t.Fatal(err) } if len(info.Peers) != 1 || info.Peers[0].NetAddress != peer.Address() { t.Fatal("/gateway/connect did not connect to peer", peer.Address()) } } // TestGatewayPeerDisconnect checks that /gateway/disconnect removes the // correct peer from the gateway's peerlist. func TestGatewayPeerDisconnect(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestGatewayPeerDisconnect1") if err != nil { t.Fatal(err) } defer st.server.Close() peer, err := gateway.New("localhost:0", false, build.TempDir("api", "TestGatewayPeerDisconnect2", "gateway")) if err != nil { t.Fatal(err) } err = st.stdPostAPI("/gateway/connect/"+string(peer.Address()), nil) if err != nil { t.Fatal(err) } var info GatewayGET st.getAPI("/gateway", &info) if len(info.Peers) != 1 || info.Peers[0].NetAddress != peer.Address() { t.Fatal("/gateway/connect did not connect to peer", peer.Address()) } err = st.stdPostAPI("/gateway/disconnect/"+string(peer.Address()), nil) if err != nil { t.Fatal(err) } err = st.getAPI("/gateway", &info) if err != nil { t.Fatal(err) } if len(info.Peers) != 0 { t.Fatal("/gateway/disconnect did not disconnect from peer", peer.Address()) } } Sia-1.0.4/api/host.go000066400000000000000000000155351302033070500143070ustar00rootroot00000000000000package api import ( "errors" "fmt" "net/http" "github.com/NebulousLabs/Sia/modules" "github.com/julienschmidt/httprouter" ) var ( // errNoPath is returned when a call fails to provide a nonempty string // for the path parameter. errNoPath = Error{"path parameter is required"} // errStorageFolderNotFound is returned if a call is made looking for a // storage folder which does not appear to exist within the storage // manager. errStorageFolderNotFound = errors.New("storage folder with the provided path could not be found") ) type ( // HostGET contains the information that is returned after a GET request to // /host - a bunch of information about the status of the host. HostGET struct { ExternalSettings modules.HostExternalSettings `json:"externalsettings"` FinancialMetrics modules.HostFinancialMetrics `json:"financialmetrics"` InternalSettings modules.HostInternalSettings `json:"internalsettings"` NetworkMetrics modules.HostNetworkMetrics `json:"networkmetrics"` } // StorageGET contains the information that is returned after a GET request // to /host/storage - a bunch of information about the status of storage // management on the host. StorageGET struct { Folders []modules.StorageFolderMetadata `json:"folders"` } ) // folderIndex determines the index of the storage folder with the provided // path. func folderIndex(folderPath string, storageFolders []modules.StorageFolderMetadata) (int, error) { for i, sf := range storageFolders { if sf.Path == folderPath { return i, nil } } return -1, errStorageFolderNotFound } // hostHandlerGET handles GET requests to the /host API endpoint, returning key // information about the host. func (api *API) hostHandlerGET(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { es := api.host.ExternalSettings() fm := api.host.FinancialMetrics() is := api.host.InternalSettings() nm := api.host.NetworkMetrics() hg := HostGET{ ExternalSettings: es, FinancialMetrics: fm, InternalSettings: is, NetworkMetrics: nm, } WriteJSON(w, hg) } // hostHandlerPOST handles POST request to the /host API endpoint, which sets // the internal settings of the host. func (api *API) hostHandlerPOST(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { // Map each query string to a field in the host settings. settings := api.host.InternalSettings() qsVars := map[string]interface{}{ "acceptingcontracts": &settings.AcceptingContracts, "maxduration": &settings.MaxDuration, "maxdownloadbatchsize": &settings.MaxDownloadBatchSize, "maxrevisebatchsize": &settings.MaxReviseBatchSize, "netaddress": &settings.NetAddress, "windowsize": &settings.WindowSize, "collateral": &settings.Collateral, "collateralbudget": &settings.CollateralBudget, "maxcollateral": &settings.MaxCollateral, "mincontractprice": &settings.MinContractPrice, "mindownloadbandwidthprice": &settings.MinDownloadBandwidthPrice, "minstorageprice": &settings.MinStoragePrice, "minuploadbandwidthprice": &settings.MinUploadBandwidthPrice, } // Iterate through the query string and replace any fields that have been // altered. for qs := range qsVars { if req.FormValue(qs) != "" { // skip empty values _, err := fmt.Sscan(req.FormValue(qs), qsVars[qs]) if err != nil { WriteError(w, Error{"Malformed " + qs}, http.StatusBadRequest) return } } } err := api.host.SetInternalSettings(settings) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } // hostAnnounceHandler handles the API call to get the host to announce itself // to the network. func (api *API) hostAnnounceHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { var err error if addr := req.FormValue("netaddress"); addr != "" { err = api.host.AnnounceAddress(modules.NetAddress(addr)) } else { err = api.host.Announce() } if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } // storageHandler returns a bunch of information about storage management on // the host. func (api *API) storageHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { WriteJSON(w, StorageGET{ Folders: api.host.StorageFolders(), }) } // storageFoldersAddHandler adds a storage folder to the storage manager. func (api *API) storageFoldersAddHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { folderPath := req.FormValue("path") var folderSize uint64 _, err := fmt.Sscan(req.FormValue("size"), &folderSize) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } err = api.host.AddStorageFolder(folderPath, folderSize) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } // storageFoldersResizeHandler resizes a storage folder in the storage manager. func (api *API) storageFoldersResizeHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { folderPath := req.FormValue("path") if folderPath == "" { WriteError(w, Error{"path parameter is required"}, http.StatusBadRequest) return } storageFolders := api.host.StorageFolders() folderIndex, err := folderIndex(folderPath, storageFolders) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } var newSize uint64 _, err = fmt.Sscan(req.FormValue("newsize"), &newSize) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } err = api.host.ResizeStorageFolder(folderIndex, newSize) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } // storageFoldersRemoveHandler removes a storage folder from the storage // manager. func (api *API) storageFoldersRemoveHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { folderPath := req.FormValue("path") if folderPath == "" { WriteError(w, Error{"path parameter is required"}, http.StatusBadRequest) return } storageFolders := api.host.StorageFolders() folderIndex, err := folderIndex(folderPath, storageFolders) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } force := req.FormValue("force") == "true" err = api.host.RemoveStorageFolder(folderIndex, force) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } // storageSectorsDeleteHandler handles the call to delete a sector from the // storage manager. func (api *API) storageSectorsDeleteHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { sectorRoot, err := scanHash(ps.ByName("merkleroot")) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } err = api.host.DeleteSector(sectorRoot) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } Sia-1.0.4/api/host_test.go000066400000000000000000000553641302033070500153520ustar00rootroot00000000000000package api import ( "io" "net/url" "path/filepath" "strconv" "testing" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/host/storagemanager" ) var ( // Various folder sizes for testing host storage folder resizing. // Must be provided as strings to the API call. minFolderSizeString = strconv.FormatUint(storagemanager.MinimumStorageFolderSize(), 10) maxFolderSizeString = strconv.FormatUint(storagemanager.MaximumStorageFolderSize(), 10) tooSmallFolderString = strconv.FormatUint(storagemanager.MinimumStorageFolderSize()-1, 10) tooLargeFolderString = strconv.FormatUint(storagemanager.MaximumStorageFolderSize()+1, 10) mediumSizeFolderString = strconv.FormatUint(3*storagemanager.MinimumStorageFolderSize(), 10) // Test cases for resizing a host's storage folder. // Running all the invalid cases before the valid ones simplifies some // logic in the tests that use resizeTests. resizeTests = []struct { sizeString string size uint64 err error }{ // invalid sizes {"", 0, io.EOF}, {"0", 0, storagemanager.ErrSmallStorageFolder}, {tooSmallFolderString, storagemanager.MinimumStorageFolderSize() - 1, storagemanager.ErrSmallStorageFolder}, {tooLargeFolderString, storagemanager.MaximumStorageFolderSize() + 1, storagemanager.ErrLargeStorageFolder}, // valid sizes {minFolderSizeString, storagemanager.MinimumStorageFolderSize(), nil}, {maxFolderSizeString, storagemanager.MaximumStorageFolderSize(), nil}, {mediumSizeFolderString, 3 * storagemanager.MinimumStorageFolderSize(), nil}, } ) // TestStorageHandler tests that host storage is being reported correctly. func TestStorageHandler(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestStorageHandler") if err != nil { t.Fatal(err) } defer st.server.Close() // Announce the host and start accepting contracts. if err := st.announceHost(); err != nil { t.Fatal(err) } if err := st.acceptContracts(); err != nil { t.Fatal(err) } if err := st.setHostStorage(); err != nil { t.Fatal(err) } // Set an allowance for the renter, allowing a contract to be formed. allowanceValues := url.Values{} allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", testPeriod) if err = st.stdPostAPI("/renter", allowanceValues); err != nil { t.Fatal(err) } // Create a file. path := filepath.Join(st.dir, "test.dat") fileBytes := 1024 if err := createRandFile(path, fileBytes); err != nil { t.Fatal(err) } // Upload to host. uploadValues := url.Values{} uploadValues.Set("source", path) if err := st.stdPostAPI("/renter/upload/test", uploadValues); err != nil { t.Fatal(err) } // Only one piece will be uploaded (10% at current redundancy) var rf RenterFiles for i := 0; i < 200 && (len(rf.Files) != 1 || rf.Files[0].UploadProgress < 10); i++ { st.getAPI("/renter/files", &rf) time.Sleep(50 * time.Millisecond) } if len(rf.Files) != 1 || rf.Files[0].UploadProgress < 10 { t.Error(rf.Files[0].UploadProgress) t.Fatal("uploading has failed") } var sg StorageGET if err := st.getAPI("/host/storage", &sg); err != nil { t.Fatal(err) } // Uploading succeeded, so /host/storage should be reporting a successful // write. if sg.Folders[0].SuccessfulWrites != 1 { t.Fatalf("expected 1 successful write, got %v", sg.Folders[0].SuccessfulWrites) } if used := sg.Folders[0].Capacity - sg.Folders[0].CapacityRemaining; used != modules.SectorSize { t.Fatalf("expected used capacity to be the size of one sector (%v bytes), got %v bytes", modules.SectorSize, used) } } // TestAddFolderNoPath tests that an API call to add a storage folder fails if // no path was provided. func TestAddFolderNoPath(t *testing.T) { if testing.Short() { t.Skip() } st, err := createServerTester("TestAddFolderNoPath") if err != nil { t.Fatal(err) } defer st.server.Close() // Try adding a storage folder without setting "path" in the API call. addValues := url.Values{} addValues.Set("size", mediumSizeFolderString) err = st.stdPostAPI("/host/storage/folders/add", addValues) if err == nil || err.Error() != storagemanager.ErrEmptyPath.Error() { t.Fatalf("expected error to be %v; got %v", storagemanager.ErrEmptyPath, err) } // Setting the path to an empty string should trigger the same error. addValues.Set("path", "") err = st.stdPostAPI("/host/storage/folders/add", addValues) if err == nil || err.Error() != storagemanager.ErrEmptyPath.Error() { t.Fatalf("expected error to be %v; got %v", storagemanager.ErrEmptyPath, err) } } // TestAddFolderNoSize tests that an API call to add a storage folder fails if // no path was provided. func TestAddFolderNoSize(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestAddFolderNoSize") if err != nil { t.Fatal(err) } defer st.server.Close() // Try adding a storage folder without setting "size" in the API call. addValues := url.Values{} addValues.Set("path", st.dir) err = st.stdPostAPI("/host/storage/folders/add", addValues) if err == nil || err.Error() != io.EOF.Error() { t.Fatalf("expected error to be %v, got %v", io.EOF, err) } } // TestAddSameFolderTwice tests that an API call that attempts to add a // host storage folder that's already been added is handled gracefully. func TestAddSameFolderTwice(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestAddSameFolderTwice") if err != nil { t.Fatal(err) } defer st.server.Close() // Make the call to add a storage folder twice. addValues := url.Values{} addValues.Set("path", st.dir) addValues.Set("size", mediumSizeFolderString) err = st.stdPostAPI("/host/storage/folders/add", addValues) if err != nil { t.Fatal(err) } err = st.stdPostAPI("/host/storage/folders/add", addValues) if err == nil || err.Error() != storagemanager.ErrRepeatFolder.Error() { t.Fatalf("expected err to be %v, got %v", err, storagemanager.ErrRepeatFolder) } } // TestResizeEmptyStorageFolder tests that invalid and valid calls to resize // an empty storage folder are properly handled. func TestResizeEmptyStorageFolder(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestResizeEmptyStorageFolder") if err != nil { t.Fatal(err) } defer st.server.Close() // Announce the host and start accepting contracts. if err := st.announceHost(); err != nil { t.Fatal(err) } if err := st.acceptContracts(); err != nil { t.Fatal(err) } if err := st.setHostStorage(); err != nil { t.Fatal(err) } // Find out how large the host's initial storage folder is. var sg StorageGET if err := st.getAPI("/host/storage", &sg); err != nil { t.Fatal(err) } defaultSize := sg.Folders[0].Capacity // Convert defaultSize (uint64) to a string for the API call. defaultSizeString := strconv.FormatUint(defaultSize, 10) resizeValues := url.Values{} resizeValues.Set("path", st.dir) resizeValues.Set("newsize", defaultSizeString) // Attempting to resize to the same size should return an error. err = st.stdPostAPI("/host/storage/folders/resize", resizeValues) if err == nil || err.Error() != storagemanager.ErrNoResize.Error() { t.Fatalf("expected error %v, got %v", storagemanager.ErrNoResize, err) } // Try resizing to a bunch of sizes (invalid ones first, valid ones second). // This ordering simplifies logic within the for loop. for _, test := range resizeTests { // Attempt to resize the host's storage folder. resizeValues.Set("newsize", test.sizeString) err = st.stdPostAPI("/host/storage/folders/resize", resizeValues) if (err == nil && test.err != nil) || (err != nil && err.Error() != test.err.Error()) { t.Fatalf("expected error to be %v, got %v", test.err, err) } // Find out if the resize call worked as expected. if err := st.getAPI("/host/storage", &sg); err != nil { t.Fatal(err) } // If the test size is valid, check that the folder has been resized // properly. if test.err == nil { // Check that the folder's total capacity has been updated. if got := sg.Folders[0].Capacity; got != test.size { t.Fatalf("expected folder to be resized to %v; got %v instead", test.size, got) } // Check that the folder's remaining capacity has been updated. if got := sg.Folders[0].CapacityRemaining; got != test.size { t.Fatalf("folder should be empty, but capacity remaining (%v) != total capacity (%v)", got, test.size) } } else { // If the test size is invalid, the folder should not have been // resized. The invalid test cases are all run before the valid ones, // so the folder size should still be defaultSize. if got := sg.Folders[0].Capacity; got != defaultSize { t.Fatalf("folder was resized to an invalid size (%v) in a test case that should have failed: %v", got, test) } } } } // TestResizeNonemptyStorageFolder tests that invalid and valid calls to resize // a storage folder with one sector filled are properly handled. // Ideally, we would also test a very full storage folder (including the case // where the host tries to resize to a size smaller than the amount of data // in the folder), but that would be a very expensive test. func TestResizeNonemptyStorageFolder(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestResizeNonemptyStorageFolder") if err != nil { t.Fatal(err) } defer st.server.Close() // Announce the host and start accepting contracts. if err := st.announceHost(); err != nil { t.Fatal(err) } if err := st.acceptContracts(); err != nil { t.Fatal(err) } if err := st.setHostStorage(); err != nil { t.Fatal(err) } // Set an allowance for the renter, allowing a contract to be formed. allowanceValues := url.Values{} allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", testPeriod) if err = st.stdPostAPI("/renter", allowanceValues); err != nil { t.Fatal(err) } // Create a file. path := filepath.Join(st.dir, "test.dat") fileBytes := 1024 if err := createRandFile(path, fileBytes); err != nil { t.Fatal(err) } // Upload to host. uploadValues := url.Values{} uploadValues.Set("source", path) if err := st.stdPostAPI("/renter/upload/test", uploadValues); err != nil { t.Fatal(err) } // Only one piece will be uploaded (10% at current redundancy) var rf RenterFiles for i := 0; i < 200 && (len(rf.Files) != 1 || rf.Files[0].UploadProgress < 10); i++ { st.getAPI("/renter/files", &rf) time.Sleep(50 * time.Millisecond) } if len(rf.Files) != 1 || rf.Files[0].UploadProgress < 10 { t.Error(rf.Files[0].UploadProgress) t.Fatal("uploading has failed") } // Find out how large the host's initial storage folder is. var sg StorageGET if err := st.getAPI("/host/storage", &sg); err != nil { t.Fatal(err) } defaultSize := sg.Folders[0].Capacity // Convert defaultSize (uint64) to a string for the API call. defaultSizeString := strconv.FormatUint(defaultSize, 10) resizeValues := url.Values{} resizeValues.Set("path", st.dir) resizeValues.Set("newsize", defaultSizeString) // Attempting to resize to the same size should return an error. err = st.stdPostAPI("/host/storage/folders/resize", resizeValues) if err == nil || err.Error() != storagemanager.ErrNoResize.Error() { t.Fatalf("expected error %v, got %v", storagemanager.ErrNoResize, err) } // Try resizing to a bunch of sizes (invalid ones first, valid ones second). // This ordering simplifies logic within the for loop. for _, test := range resizeTests { // Attempt to resize the host's storage folder. resizeValues.Set("newsize", test.sizeString) err = st.stdPostAPI("/host/storage/folders/resize", resizeValues) if (err == nil && test.err != nil) || (err != nil && err.Error() != test.err.Error()) { t.Fatalf("expected error to be %v, got %v", test.err, err) } // Find out if the resize call worked as expected. if err := st.getAPI("/host/storage", &sg); err != nil { t.Fatal(err) } // If the test size is valid, check that the folder has been resized // properly. if test.err == nil { // Check that the folder's total capacity has been updated. if sg.Folders[0].Capacity != test.size { t.Fatalf("expected folder to be resized to %v; got %v instead", test.size, sg.Folders[0].Capacity) } // Since one sector has been uploaded, the available capacity // should be one sector size smaller than the total capacity. if used := test.size - sg.Folders[0].CapacityRemaining; used != modules.SectorSize { t.Fatalf("used capacity (%v) != the size of 1 sector (%v)", used, modules.SectorSize) } } else { // If the test size is invalid, the folder should not have been // resized. The invalid test cases are all run before the valid // ones, so the folder size should still be defaultSize. if got := sg.Folders[0].Capacity; got != defaultSize { t.Fatalf("folder was resized to an invalid size (%v) in a test case that should have failed: %v", got, test) } } } } // TestResizeNonexistentFolder checks that an API call to resize a nonexistent // folder triggers the appropriate error. func TestResizeNonexistentFolder(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestResizeNonexistentFolder") if err != nil { t.Fatal(err) } defer st.server.Close() // No folder has been created yet at st.dir, so using it as the path for // the resize call should trigger an error. resizeValues := url.Values{} resizeValues.Set("path", st.dir) resizeValues.Set("newsize", mediumSizeFolderString) err = st.stdPostAPI("/host/storage/folders/resize", resizeValues) if err == nil || err.Error() != errStorageFolderNotFound.Error() { t.Fatalf("expected error to be %v, got %v", errStorageFolderNotFound, err) } } // TestResizeFolderNoPath checks that an API call to resize a storage folder fails // if no path was provided. func TestResizeFolderNoPath(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestResizeFolderNoPath") if err != nil { t.Fatal(err) } defer st.server.Close() // The call to resize should fail if no path has been provided. resizeValues := url.Values{} resizeValues.Set("newsize", mediumSizeFolderString) err = st.stdPostAPI("/host/storage/folders/resize", resizeValues) if err == nil || err.Error() != errNoPath.Error() { t.Fatalf("expected error to be %v; got %v", errNoPath, err) } } // TestRemoveEmptyStorageFolder checks that removing an empty storage folder // succeeds -- even if the host is left with zero storage folders. func TestRemoveEmptyStorageFolder(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRemoveEmptyStorageFolder") if err != nil { t.Fatal(err) } defer st.server.Close() // Set up a storage folder for the host. if err := st.setHostStorage(); err != nil { t.Fatal(err) } // Try to delete the host's empty storage folder. removeValues := url.Values{} removeValues.Set("path", st.dir) if err = st.stdPostAPI("/host/storage/folders/remove", removeValues); err != nil { t.Fatal(err) } } // TestRemoveStorageFolderError checks that invalid calls to // /host/storage/folders/remove fail with the appropriate error. func TestRemoveStorageFolderError(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRemoveStorageFolderErr") if err != nil { t.Fatal(err) } defer st.server.Close() // Set up a storage folder for the host. if err := st.setHostStorage(); err != nil { t.Fatal(err) } // Try removing a nonexistent folder. removeValues := url.Values{} removeValues.Set("path", "/foo/bar") err = st.stdPostAPI("/host/storage/folders/remove", removeValues) if err == nil || err.Error() != errStorageFolderNotFound.Error() { t.Fatalf("expected error %v, got %v", errStorageFolderNotFound, err) } // The folder path can't be an empty string. removeValues.Set("path", "") err = st.stdPostAPI("/host/storage/folders/remove", removeValues) if err == nil || err.Error() != errNoPath.Error() { t.Fatalf("expected error to be %v; got %v", errNoPath, err) } } // TestRemoveStorageFolderForced checks that if a call to remove a storage // folder will result in data loss, that call succeeds if and only if "force" // has been set to "true". func TestRemoveStorageFolderForced(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRemoveStorageFolderForced") if err != nil { t.Fatal(err) } defer st.server.Close() // Announce the host. if err := st.announceHost(); err != nil { t.Fatal(err) } if err := st.acceptContracts(); err != nil { t.Fatal(err) } if err := st.setHostStorage(); err != nil { t.Fatal(err) } // Set an allowance for the renter, allowing a contract to be formed. allowanceValues := url.Values{} allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", testPeriod) if err = st.stdPostAPI("/renter", allowanceValues); err != nil { t.Fatal(err) } // Create a file for upload. path := filepath.Join(st.dir, "test.dat") if err := createRandFile(path, 512); err != nil { t.Fatal(err) } // Upload to host. uploadValues := url.Values{} uploadValues.Set("source", path) if err := st.stdPostAPI("/renter/upload/test", uploadValues); err != nil { t.Fatal(err) } // Only one piece will be uploaded (10% at current redundancy) var rf RenterFiles for i := 0; i < 200 && (len(rf.Files) != 1 || rf.Files[0].UploadProgress < 10); i++ { st.getAPI("/renter/files", &rf) time.Sleep(50 * time.Millisecond) } if len(rf.Files) != 1 || rf.Files[0].UploadProgress < 10 { t.Error(rf.Files[0].UploadProgress) t.Fatal("uploading has failed") } // The host should not be able to remove its only folder without setting // "force" to "true", since this will result in data loss (there are no // other folders for the data to be redistributed to). removeValues := url.Values{} removeValues.Set("path", st.dir) err = st.stdPostAPI("/host/storage/folders/remove", removeValues) if err == nil || err.Error() != storagemanager.ErrIncompleteOffload.Error() { t.Fatalf("expected err to be %v; got %v", storagemanager.ErrIncompleteOffload, err) } // Forced removal of the folder should succeed, though. removeValues.Set("force", "true") err = st.stdPostAPI("/host/storage/folders/remove", removeValues) if err != nil { t.Fatal(err) } } // TestDeleteSector tests the call to delete a storage sector from the host. func TestDeleteSector(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestDeleteSector") if err != nil { t.Fatal(err) } defer st.server.Close() // Set up the host for forming contracts. if err := st.announceHost(); err != nil { t.Fatal(err) } if err := st.acceptContracts(); err != nil { t.Fatal(err) } if err := st.setHostStorage(); err != nil { t.Fatal(err) } // Set an allowance for the renter, allowing contracts to formed. allowanceValues := url.Values{} allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", testPeriod) if err = st.stdPostAPI("/renter", allowanceValues); err != nil { t.Fatal(err) } // Create a file. path := filepath.Join(st.dir, "test.dat") if err := createRandFile(path, 1024); err != nil { t.Fatal(err) } // Upload to host. uploadValues := url.Values{} uploadValues.Set("source", path) if err = st.stdPostAPI("/renter/upload/test", uploadValues); err != nil { t.Fatal(err) } // Only one piece will be uploaded (10% at current redundancy) var rf RenterFiles for i := 0; i < 200 && (len(rf.Files) != 1 || rf.Files[0].UploadProgress < 10); i++ { st.getAPI("/renter/files", &rf) time.Sleep(50 * time.Millisecond) } if len(rf.Files) != 1 || rf.Files[0].UploadProgress < 10 { t.Error(rf.Files[0].UploadProgress) t.Fatal("uploading has failed") } // Get the Merkle root of the piece that was uploaded. contracts := st.renter.Contracts() if len(contracts) != 1 { t.Fatalf("expected exactly 1 contract to have been formed; got %v instead", len(contracts)) } sectorRoot := contracts[0].MerkleRoots[0].String() if err = st.stdPostAPI("/host/storage/sectors/delete/"+sectorRoot, url.Values{}); err != nil { t.Fatal(err) } } // TestDeleteNonexistentSector checks that attempting to delete a storage // sector that doesn't exist will fail with the appropriate error. func TestDeleteNonexistentSector(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestDeleteNonexistentSector") if err != nil { t.Fatal(err) } defer st.server.Close() // These calls to delete imaginary sectors should fail for a few reasons: // - the given sector root strings are invalid // - the renter hasn't uploaded anything // - the host has no storage folders yet // Right now, the calls fail for the first reason. This test will report if that behavior changes. badHash := crypto.HashObject("fake object").String() err = st.stdPostAPI("/host/storage/sectors/delete/"+badHash, url.Values{}) if err == nil || err.Error() != storagemanager.ErrSectorNotFound.Error() { t.Fatalf("expected error to be %v; got %v", storagemanager.ErrSectorNotFound, err) } wrongSize := "wrong size string" err = st.stdPostAPI("/host/storage/sectors/delete/"+wrongSize, url.Values{}) if err == nil || err.Error() != crypto.ErrHashWrongLen.Error() { t.Fatalf("expected error to be %v; got %v", crypto.ErrHashWrongLen, err) } } /* // TestIntegrationRenewing tests that the renter and host manage contract // renewals properly. func TestIntegrationRenewing(t *testing.T) { st, err := createServerTester("TestIntegrationRenewing") if err != nil { t.Fatal(err) } defer st.server.Close() // Announce the host. err = st.announceHost() if err != nil { t.Fatal(err) } // create a file path := filepath.Join(build.SiaTestingDir, "api", "TestIntegrationRenewing", "test.dat") err = createRandFile(path) if err != nil { t.Fatal(err) } // upload to host, specifying that the file should be renewed uploadValues := url.Values{} uploadValues.Set("source", path) err = st.stdPostAPI("/renter/upload/test", uploadValues) if err != nil { t.Fatal(err) } // only one piece will be uploaded (10% at current redundancy) var rf RenterFiles for i := 0; i < 150 && (len(rf.Files) != 1 || rf.Files[0].UploadProgress != 10); i++ { st.getAPI("/renter/files", &rf) time.Sleep(50 * time.Millisecond) } if len(rf.Files) != 1 || rf.Files[0].UploadProgress != 10 { t.Error(rf.Files[0].UploadProgress) t.Fatal("uploading has failed") } // default expiration is 20 blocks expExpiration := st.cs.Height() + 20 if rf.Files[0].Expiration != expExpiration { t.Fatalf("expected expiration of %v, got %v", expExpiration, rf.Files[0].Expiration) } // mine blocks until we hit the renew threshold (default 10 blocks) for st.cs.Height() < expExpiration-10 { st.miner.AddBlock() } // renter should now renew the contract for another 20 blocks newExpiration := st.cs.Height() + 20 for i := 0; i < 5 && rf.Files[0].Expiration != newExpiration; i++ { time.Sleep(1 * time.Second) st.getAPI("/renter/files", &rf) } } */ Sia-1.0.4/api/miner.go000066400000000000000000000041601302033070500144340ustar00rootroot00000000000000package api import ( "net/http" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/types" "github.com/julienschmidt/httprouter" ) type ( // MinerGET contains the information that is returned after a GET request // to /miner. MinerGET struct { BlocksMined int `json:"blocksmined"` CPUHashrate int `json:"cpuhashrate"` CPUMining bool `json:"cpumining"` StaleBlocksMined int `json:"staleblocksmined"` } ) // minerHandler handles the API call that queries the miner's status. func (api *API) minerHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { blocksMined, staleMined := api.miner.BlocksMined() mg := MinerGET{ BlocksMined: blocksMined, CPUHashrate: api.miner.CPUHashrate(), CPUMining: api.miner.CPUMining(), StaleBlocksMined: staleMined, } WriteJSON(w, mg) } // minerStartHandler handles the API call that starts the miner. func (api *API) minerStartHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { api.miner.StartCPUMining() WriteSuccess(w) } // minerStopHandler handles the API call to stop the miner. func (api *API) minerStopHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { api.miner.StopCPUMining() WriteSuccess(w) } // minerHeaderHandlerGET handles the API call that retrieves a block header // for work. func (api *API) minerHeaderHandlerGET(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { bhfw, target, err := api.miner.HeaderForWork() if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } w.Write(encoding.MarshalAll(target, bhfw)) } // minerHeaderHandlerPOST handles the API call to submit a block header to the // miner. func (api *API) minerHeaderHandlerPOST(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { var bh types.BlockHeader err := encoding.NewDecoder(req.Body).Decode(&bh) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } err = api.miner.SubmitHeader(bh) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } Sia-1.0.4/api/miner_test.go000066400000000000000000000076621302033070500155050ustar00rootroot00000000000000package api import ( "io/ioutil" "testing" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/types" ) // TestIntegrationMinerGET checks the GET call to the /miner endpoint. func TestIntegrationMinerGET(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestIntegrationMinerGET") if err != nil { t.Fatal(err) } defer st.server.Close() // Get the api returned fields of the miner. var mg MinerGET err = st.getAPI("/miner", &mg) if err != nil { t.Fatal(err) } // Verify the correctness of the results. blocksMined, staleBlocksMined := st.server.api.miner.BlocksMined() if mg.BlocksMined != blocksMined { t.Error("blocks mined did not succeed") } if mg.StaleBlocksMined != staleBlocksMined { t.Error("stale blocks mined is incorrect") } if mg.CPUHashrate != st.server.api.miner.CPUHashrate() { t.Error("mismatched cpu hashrate") } if mg.CPUMining != st.server.api.miner.CPUMining() { t.Error("mismatched cpu miner status") } } // TestIntegrationMinerStartStop checks that the miner start and miner stop api endpoints // toggle the cpu miner. func TestIntegrationMinerStartStop(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestIntegrationMinerStartStop") if err != nil { t.Fatal(err) } defer st.server.Close() // Start the cpu miner, give time for the first hashrate readings to // appear. err = st.stdGetAPI("/miner/start") if err != nil { t.Fatal(err) } time.Sleep(100 * time.Millisecond) if !st.server.api.miner.CPUMining() { t.Error("cpu miner is reporting that it is not on") } // Check the numbers through the status api call. var mg MinerGET err = st.getAPI("/miner", &mg) if err != nil { t.Fatal(err) } if !mg.CPUMining { t.Error("cpu is not reporting through the api that it is mining") } // Stop the cpu miner and wait for the stop call to go through. err = st.stdGetAPI("/miner/stop") if err != nil { t.Fatal(err) } time.Sleep(100 * time.Millisecond) if st.server.api.miner.CPUMining() { t.Error("cpu miner is reporting that it is on after being stopped") } // Check the numbers through the status api call. err = st.getAPI("/miner", &mg) if err != nil { t.Fatal(err) } if mg.CPUMining { t.Error("cpu is not reporting through the api that it is mining") } } // TestIntegrationMinerHeader checks that the header GET and POST calls are // useful tools for mining blocks. func TestIntegrationMinerHeader(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestIntegrationMinerHeader") if err != nil { t.Fatal(err) } defer st.server.Close() startingHeight := st.cs.Height() // Get a header that can be used for mining. resp, err := HttpGET("http://" + st.server.listener.Addr().String() + "/miner/header") if err != nil { t.Fatal(err) } defer resp.Body.Close() targetAndHeader, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } // Twiddle the header bits until a block has been found. // // Note: this test treats the target as hardcoded, if the testing target is // changed, this test will also need to be changed. if types.RootTarget[0] != 128 { t.Fatal("test will fail because the testing constants have been unexpectedly changed") } var header [80]byte copy(header[:], targetAndHeader[32:]) headerHash := crypto.HashObject(header) for headerHash[0] >= types.RootTarget[0] { header[35]++ headerHash = crypto.HashObject(header) } // Submit the solved header through the api and check that the height of // the blockchain increases. resp, err = HttpPOST("http://"+st.server.listener.Addr().String()+"/miner/header", string(header[:])) if err != nil { t.Fatal(err) } defer resp.Body.Close() time.Sleep(500 * time.Millisecond) if st.cs.Height() != startingHeight+1 { t.Errorf("block height did not increase after trying to mine a block through the api, started at %v and ended at %v", startingHeight, st.cs.Height()) } } Sia-1.0.4/api/renter.go000066400000000000000000000227371302033070500146330ustar00rootroot00000000000000package api import ( "fmt" "net/http" "path/filepath" "strings" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/julienschmidt/httprouter" ) var ( // TODO: Replace this function by accepting user input. recommendedHosts = func() uint64 { if build.Release == "dev" { return 2 } if build.Release == "standard" { return 24 } if build.Release == "testing" { return 1 } panic("unrecognized release constant in api") }() ) type ( // RenterGET contains various renter metrics. RenterGET struct { Settings modules.RenterSettings `json:"settings"` FinancialMetrics modules.RenterFinancialMetrics `json:"financialmetrics"` } // RenterContract represents a contract formed by the renter. RenterContract struct { EndHeight types.BlockHeight `json:"endheight"` ID types.FileContractID `json:"id"` NetAddress modules.NetAddress `json:"netaddress"` RenterFunds types.Currency `json:"renterfunds"` Size uint64 `json:"size"` } // RenterContracts contains the renter's contracts. RenterContracts struct { Contracts []RenterContract `json:"contracts"` } // DownloadQueue contains the renter's download queue. RenterDownloadQueue struct { Downloads []modules.DownloadInfo `json:"downloads"` } // RenterFiles lists the files known to the renter. RenterFiles struct { Files []modules.FileInfo `json:"files"` } // RenterLoad lists files that were loaded into the renter. RenterLoad struct { FilesAdded []string `json:"filesadded"` } // RenterShareASCII contains an ASCII-encoded .sia file. RenterShareASCII struct { ASCIIsia string `json:"asciisia"` } // ActiveHosts lists active hosts on the network. ActiveHosts struct { Hosts []modules.HostDBEntry `json:"hosts"` } // AllHosts lists all hosts that the renter is aware of. AllHosts struct { Hosts []modules.HostDBEntry `json:"hosts"` } ) // renterHandlerGET handles the API call to /renter. func (api *API) renterHandlerGET(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { WriteJSON(w, RenterGET{ Settings: api.renter.Settings(), FinancialMetrics: api.renter.FinancialMetrics(), }) } // renterHandlerPOST handles the API call to set the Renter's settings. func (api *API) renterHandlerPOST(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { // scan values funds, ok := scanAmount(req.FormValue("funds")) if !ok { WriteError(w, Error{"Couldn't parse funds"}, http.StatusBadRequest) return } // var hosts uint64 // _, err := fmt.Sscan(req.FormValue("hosts"), &hosts) // if err != nil { // WriteError(w, Error{"Couldn't parse hosts: "+err.Error()}, http.StatusBadRequest) // return // } var period types.BlockHeight _, err := fmt.Sscan(req.FormValue("period"), &period) if err != nil { WriteError(w, Error{"Couldn't parse period: " + err.Error()}, http.StatusBadRequest) return } // var renewWindow types.BlockHeight // _, err = fmt.Sscan(req.FormValue("renewwindow"), &renewWindow) // if err != nil { // WriteError(w, Error{"Couldn't parse renewwindow: "+err.Error()}, http.StatusBadRequest) // return // } err = api.renter.SetSettings(modules.RenterSettings{ Allowance: modules.Allowance{ Funds: funds, Period: period, // TODO: let user specify these Hosts: recommendedHosts, RenewWindow: period / 2, }, }) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } // renterContractsHandler handles the API call to request the Renter's contracts. func (api *API) renterContractsHandler(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { contracts := []RenterContract{} for _, c := range api.renter.Contracts() { contracts = append(contracts, RenterContract{ EndHeight: c.EndHeight(), ID: c.ID, NetAddress: c.NetAddress, RenterFunds: c.RenterFunds(), Size: modules.SectorSize * uint64(len(c.MerkleRoots)), }) } WriteJSON(w, RenterContracts{ Contracts: contracts, }) } // renterDownloadsHandler handles the API call to request the download queue. func (api *API) renterDownloadsHandler(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { WriteJSON(w, RenterDownloadQueue{ Downloads: api.renter.DownloadQueue(), }) } // renterLoadHandler handles the API call to load a '.sia' file. func (api *API) renterLoadHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { source := req.FormValue("source") if !filepath.IsAbs(source) { WriteError(w, Error{"source must be an absolute path"}, http.StatusBadRequest) return } files, err := api.renter.LoadSharedFiles(source) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteJSON(w, RenterLoad{FilesAdded: files}) } // renterLoadAsciiHandler handles the API call to load a '.sia' file // in ASCII form. func (api *API) renterLoadAsciiHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { files, err := api.renter.LoadSharedFilesAscii(req.FormValue("asciisia")) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteJSON(w, RenterLoad{FilesAdded: files}) } // renterRenameHandler handles the API call to rename a file entry in the // renter. func (api *API) renterRenameHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { err := api.renter.RenameFile(strings.TrimPrefix(ps.ByName("siapath"), "/"), req.FormValue("newsiapath")) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } // renterFilesHandler handles the API call to list all of the files. func (api *API) renterFilesHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { WriteJSON(w, RenterFiles{ Files: api.renter.FileList(), }) } // renterDeleteHandler handles the API call to delete a file entry from the // renter. func (api *API) renterDeleteHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { err := api.renter.DeleteFile(strings.TrimPrefix(ps.ByName("siapath"), "/")) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } // renterDownloadHandler handles the API call to download a file. func (api *API) renterDownloadHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { destination := req.FormValue("destination") // Check that the destination path is absolute. if !filepath.IsAbs(destination) { WriteError(w, Error{"destination must be an absolute path"}, http.StatusBadRequest) return } err := api.renter.Download(strings.TrimPrefix(ps.ByName("siapath"), "/"), destination) if err != nil { WriteError(w, Error{"Download failed: " + err.Error()}, http.StatusInternalServerError) return } WriteSuccess(w) } // renterShareHandler handles the API call to create a '.sia' file that // shares a set of file. func (api *API) renterShareHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { destination := req.FormValue("destination") // Check that the destination path is absolute. if !filepath.IsAbs(destination) { WriteError(w, Error{"destination must be an absolute path"}, http.StatusBadRequest) return } err := api.renter.ShareFiles(strings.Split(req.FormValue("siapaths"), ","), destination) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } // renterShareAsciiHandler handles the API call to return a '.sia' file // in ascii form. func (api *API) renterShareAsciiHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { ascii, err := api.renter.ShareFilesAscii(strings.Split(req.FormValue("siapaths"), ",")) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteJSON(w, RenterShareASCII{ ASCIIsia: ascii, }) } // renterUploadHandler handles the API call to upload a file. func (api *API) renterUploadHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { source := req.FormValue("source") if !filepath.IsAbs(source) { WriteError(w, Error{"source must be an absolute path"}, http.StatusBadRequest) return } err := api.renter.Upload(modules.FileUploadParams{ Source: source, SiaPath: strings.TrimPrefix(ps.ByName("siapath"), "/"), // let the renter decide these values; eventually they will be configurable ErasureCode: nil, }) if err != nil { WriteError(w, Error{"Upload failed: " + err.Error()}, http.StatusInternalServerError) return } WriteSuccess(w) } // renterHostsActiveHandler handles the API call asking for the list of active // hosts. func (api *API) renterHostsActiveHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { var numHosts uint64 hosts := api.renter.ActiveHosts() if req.FormValue("numhosts") == "" { // Default value for 'numhosts' is all of them. numHosts = uint64(len(hosts)) } else { // Parse the value for 'numhosts'. _, err := fmt.Sscan(req.FormValue("numhosts"), &numHosts) if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } // Catch any boundary errors. if numHosts > uint64(len(hosts)) { numHosts = uint64(len(hosts)) } } WriteJSON(w, ActiveHosts{ Hosts: hosts[:numHosts], }) } // renterHostsAllHandler handles the API call asking for the list of all hosts. func (api *API) renterHostsAllHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { WriteJSON(w, AllHosts{ Hosts: api.renter.AllHosts(), }) } Sia-1.0.4/api/renter_test.go000066400000000000000000000505601302033070500156650ustar00rootroot00000000000000package api import ( "io/ioutil" "net/url" "path/filepath" "strconv" "strings" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules/renter" "github.com/NebulousLabs/Sia/modules/renter/contractor" "github.com/NebulousLabs/Sia/types" ) const ( testFunds = "10000000000000000000000000000" testPeriod = "5" ) // createRandFile creates a file on disk and fills it with random bytes. func createRandFile(path string, size int) error { data, err := crypto.RandBytes(size) if err != nil { return err } return ioutil.WriteFile(path, data, 0600) } // TestRenterPaths tests that the /renter routes handle path parameters // properly. func TestRenterPaths(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRenterPaths") if err != nil { t.Fatal(err) } defer st.server.Close() // Announce the host. err = st.announceHost() if err != nil { t.Fatal(err) } // Create a file. path := filepath.Join(build.SiaTestingDir, "api", "TestRenterPaths", "test.dat") err = createRandFile(path, 1024) if err != nil { t.Fatal(err) } // Upload to host. uploadValues := url.Values{} uploadValues.Set("source", path) uploadValues.Set("renew", "true") err = st.stdPostAPI("/renter/upload/foo/bar/test", uploadValues) if err != nil { t.Fatal(err) } // File should be listed by the renter. var rf RenterFiles err = st.getAPI("/renter/files", &rf) if err != nil { t.Fatal(err) } if len(rf.Files) != 1 || rf.Files[0].SiaPath != "foo/bar/test" { t.Fatal("/renter/files did not return correct file:", rf) } } // TestRenterConflicts tests that the renter handles naming conflicts properly. func TestRenterConflicts(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRenterConflicts") if err != nil { t.Fatal(err) } defer st.server.Close() // Announce the host. err = st.announceHost() if err != nil { t.Fatal(err) } // Create a file. path := filepath.Join(build.SiaTestingDir, "api", "TestRenterConflicts", "test.dat") err = createRandFile(path, 1024) if err != nil { t.Fatal(err) } // Upload to host, using a path designed to cause conflicts. The renter // should automatically create a folder called foo/bar.sia. Later, we'll // exploit this by uploading a file called foo/bar. uploadValues := url.Values{} uploadValues.Set("source", path) uploadValues.Set("renew", "true") err = st.stdPostAPI("/renter/upload/foo/bar.sia/test", uploadValues) if err != nil { t.Fatal(err) } // File should be listed by the renter. var rf RenterFiles err = st.getAPI("/renter/files", &rf) if err != nil { t.Fatal(err) } if len(rf.Files) != 1 || rf.Files[0].SiaPath != "foo/bar.sia/test" { t.Fatal("/renter/files did not return correct file:", rf) } // Upload using the same nickname. err = st.stdPostAPI("/renter/upload/foo/bar.sia/test", uploadValues) expectedErr := Error{"Upload failed: " + renter.ErrPathOverload.Error()} if err != expectedErr { t.Fatalf("expected %v, got %v", Error{"Upload failed: " + renter.ErrPathOverload.Error()}, err) } // Upload using nickname that conflicts with folder. err = st.stdPostAPI("/renter/upload/foo/bar", uploadValues) if err == nil { t.Fatal("expecting conflict error, got nil") } } // TestRenterHostsActiveHandler checks the behavior of the call to // /hostdb/active. func TestRenterHostsActiveHandler(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRenterHostsActiveHandler") if err != nil { t.Fatal(err) } defer st.server.Close() // Try the call with numhosts unset, and set to -1, 0, and 1. var ah ActiveHosts err = st.getAPI("/hostdb/active", &ah) if err != nil { t.Fatal(err) } if len(ah.Hosts) != 0 { t.Fatal(len(ah.Hosts)) } err = st.getAPI("/hostdb/active?numhosts=-1", &ah) if err == nil { t.Fatal("expecting an error, got:", err) } err = st.getAPI("/hostdb/active?numhosts=0", &ah) if err != nil { t.Fatal(err) } if len(ah.Hosts) != 0 { t.Fatal(len(ah.Hosts)) } err = st.getAPI("/hostdb/active?numhosts=1", &ah) if err != nil { t.Fatal(err) } if len(ah.Hosts) != 0 { t.Fatal(len(ah.Hosts)) } // announce the host and start accepting contracts err = st.announceHost() if err != nil { t.Fatal(err) } err = st.acceptContracts() if err != nil { t.Fatal(err) } err = st.setHostStorage() if err != nil { t.Fatal(err) } // Try the call with with numhosts unset, and set to -1, 0, 1, and 2. err = st.getAPI("/hostdb/active", &ah) if err != nil { t.Fatal(err) } if len(ah.Hosts) != 1 { t.Fatal(len(ah.Hosts)) } err = st.getAPI("/hostdb/active?numhosts=-1", &ah) if err == nil { t.Fatal("expecting an error, got:", err) } err = st.getAPI("/hostdb/active?numhosts=0", &ah) if err != nil { t.Fatal(err) } if len(ah.Hosts) != 0 { t.Fatal(len(ah.Hosts)) } err = st.getAPI("/hostdb/active?numhosts=1", &ah) if err != nil { t.Fatal(err) } if len(ah.Hosts) != 1 { t.Fatal(len(ah.Hosts)) } err = st.getAPI("/hostdb/active?numhosts=2", &ah) if err != nil { t.Fatal(err) } if len(ah.Hosts) != 1 { t.Fatal(len(ah.Hosts)) } } // TestRenterHostsAllHandler checks that announcing a host adds it to the list // of all hosts. func TestRenterHostsAllHandler(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRenterHostsAllHandler") if err != nil { t.Fatal(err) } defer st.server.Close() // Try the call before any hosts have been declared. var ah AllHosts if err = st.getAPI("/hostdb/all", &ah); err != nil { t.Fatal(err) } if len(ah.Hosts) != 0 { t.Fatalf("expected 0 hosts, got %v", len(ah.Hosts)) } // Announce the host and try the call again. if err = st.announceHost(); err != nil { t.Fatal(err) } if err = st.getAPI("/hostdb/all", &ah); err != nil { t.Fatal(err) } if len(ah.Hosts) != 1 { t.Fatalf("expected 1 host, got %v", len(ah.Hosts)) } } // TestRenterHandlerContracts checks that contract formation between a host and // renter behaves as expected, and that contract spending is the right amount. func TestRenterHandlerContracts(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRenterHandlerContracts") if err != nil { t.Fatal(err) } defer st.server.Close() // Anounce the host and start accepting contracts. if err := st.announceHost(); err != nil { t.Fatal(err) } if err = st.acceptContracts(); err != nil { t.Fatal(err) } if err = st.setHostStorage(); err != nil { t.Fatal(err) } // The renter should not have any contracts yet. var contracts RenterContracts if err = st.getAPI("/renter/contracts", &contracts); err != nil { t.Fatal(err) } if len(contracts.Contracts) != 0 { t.Fatalf("expected renter to have 0 contracts; got %v", len(contracts.Contracts)) } // Set an allowance for the renter, allowing a contract to be formed. allowanceValues := url.Values{} allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", testPeriod) if err = st.stdPostAPI("/renter", allowanceValues); err != nil { t.Fatal(err) } // The renter should now have 1 contract. if err = st.getAPI("/renter/contracts", &contracts); err != nil { t.Fatal(err) } if len(contracts.Contracts) != 1 { t.Fatalf("expected renter to have 1 contract; got %v", len(contracts.Contracts)) } // Check the renter's contract spending. var get RenterGET if err = st.getAPI("/renter", &get); err != nil { t.Fatal(err) } var expectedContractSpending types.Currency for _, contract := range contracts.Contracts { expectedContractSpending = expectedContractSpending.Add(contract.RenterFunds) } if got := get.FinancialMetrics.ContractSpending; got.Cmp(expectedContractSpending) != 0 { t.Fatalf("expected contract spending to be %v; got %v", expectedContractSpending, got) } } // TestRenterHandlerGetAndPost checks that valid /renter calls successfully set // allowance values, while /renter calls with invalid allowance values are // correctly handled. func TestRenterHandlerGetAndPost(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRenterHandlerGetAndPost") if err != nil { t.Fatal(err) } defer st.server.Close() // Anounce the host and start accepting contracts. if err := st.announceHost(); err != nil { t.Fatal(err) } if err = st.acceptContracts(); err != nil { t.Fatal(err) } if err = st.setHostStorage(); err != nil { t.Fatal(err) } // Set an allowance for the renter, allowing a contract to be formed. allowanceValues := url.Values{} allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", testPeriod) if err = st.stdPostAPI("/renter", allowanceValues); err != nil { t.Fatal(err) } // Check that a call to /renter returns the expected values. var get RenterGET if err = st.getAPI("/renter", &get); err != nil { t.Fatal(err) } // Check the renter's funds. expectedFunds, ok := scanAmount(testFunds) if !ok { t.Fatal("scanAmount failed") } if got := get.Settings.Allowance.Funds; got.Cmp(expectedFunds) != 0 { t.Fatalf("expected funds to be %v; got %v", expectedFunds, got) } // Check the renter's period. intPeriod, err := strconv.Atoi(testPeriod) if err != nil { t.Fatal(err) } expectedPeriod := types.BlockHeight(intPeriod) if got := get.Settings.Allowance.Period; got != expectedPeriod { t.Fatalf("expected period to be %v; got %v", expectedPeriod, got) } // Check the renter's renew window. expectedRenewWindow := expectedPeriod / 2 if got := get.Settings.Allowance.RenewWindow; got != expectedRenewWindow { t.Fatalf("expected renew window to be %v; got %v", expectedRenewWindow, got) } // Try an empty funds string. allowanceValues = url.Values{} allowanceValues.Set("funds", "") allowanceValues.Set("period", testPeriod) err = st.stdPostAPI("/renter", allowanceValues) if err == nil || err.Error() != "Couldn't parse funds" { t.Errorf("expected error to be 'Couldn't parse funds'; got %v", err) } // Try an invalid funds string. Can't test a negative value since // ErrNegativeCurrency triggers a build.Critical, which calls a panic in // debug mode. allowanceValues.Set("funds", "0") err = st.stdPostAPI("/renter", allowanceValues) if err == nil || err.Error() != contractor.ErrInsufficientAllowance.Error() { t.Errorf("expected error to be %v; got %v", contractor.ErrInsufficientAllowance, err) } // Try a empty period string. allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", "") err = st.stdPostAPI("/renter", allowanceValues) if err == nil || !strings.HasPrefix(err.Error(), "Couldn't parse period: ") { t.Errorf("expected error to begin with 'Couldn't parse period: '; got %v", err) } // Try an invalid period string. allowanceValues.Set("period", "-1") err = st.stdPostAPI("/renter", allowanceValues) if err == nil || err.Error()[:23] != "Couldn't parse period: " { t.Errorf("expected error to begin with 'Couldn't parse period: '; got %v", err) } // Try a period that will lead to a length-zero RenewWindow. allowanceValues.Set("period", "1") err = st.stdPostAPI("/renter", allowanceValues) if err == nil || err.Error() != contractor.ErrAllowanceZeroWindow.Error() { t.Errorf("expected error to be %v, got %v", contractor.ErrAllowanceZeroWindow, err) } } // TestRenterLoadNonexistent checks that attempting to upload or download a // nonexistent file triggers the appropriate error. func TestRenterLoadNonexistent(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRenterLoadNonexistent") if err != nil { t.Fatal(err) } defer st.server.Close() // Anounce the host and start accepting contracts. if err := st.announceHost(); err != nil { t.Fatal(err) } if err = st.acceptContracts(); err != nil { t.Fatal(err) } if err = st.setHostStorage(); err != nil { t.Fatal(err) } // Set an allowance for the renter, allowing a contract to be formed. allowanceValues := url.Values{} allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", testPeriod) if err = st.stdPostAPI("/renter", allowanceValues); err != nil { t.Fatal(err) } // Try uploading a nonexistent file. fakepath := filepath.Join(st.dir, "dne.dat") uploadValues := url.Values{} uploadValues.Set("source", fakepath) err = st.stdPostAPI("/renter/upload/dne", uploadValues) if err == nil { t.Errorf("expected error when uploading nonexistent file") } // Try downloading a nonexistent file. downpath := filepath.Join(st.dir, "dnedown.dat") err = st.stdGetAPI("/renter/download/dne?destination=" + downpath) if err == nil || err.Error() != "Download failed: no file with that path" { t.Errorf("expected error to be 'Download failed: no file with that path'; got %v instead", err) } // The renter's downloads queue should be empty. var queue RenterDownloadQueue if err = st.getAPI("/renter/downloads", &queue); err != nil { t.Fatal(err) } if len(queue.Downloads) != 0 { t.Fatalf("expected renter to have 0 downloads in the queue; got %v", len(queue.Downloads)) } } // TestRenterHandlerRename checks that valid /renter/rename calls are // successful, and that invalid calls fail with the appropriate error. func TestRenterHandlerRename(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRenterHandlerRename") if err != nil { t.Fatal(err) } defer st.server.Close() // Anounce the host and start accepting contracts. if err := st.announceHost(); err != nil { t.Fatal(err) } if err = st.acceptContracts(); err != nil { t.Fatal(err) } if err = st.setHostStorage(); err != nil { t.Fatal(err) } // Set an allowance for the renter, allowing a contract to be formed. allowanceValues := url.Values{} allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", testPeriod) if err = st.stdPostAPI("/renter", allowanceValues); err != nil { t.Fatal(err) } // Create two files. path1 := filepath.Join(st.dir, "test1.dat") if err = createRandFile(path1, 512); err != nil { t.Fatal(err) } path2 := filepath.Join(st.dir, "test2.dat") if err = createRandFile(path2, 512); err != nil { t.Fatal(err) } // Upload to host. uploadValues := url.Values{} uploadValues.Set("source", path1) if err = st.stdPostAPI("/renter/upload/test1", uploadValues); err != nil { t.Fatal(err) } uploadValues.Set("source", path2) if err = st.stdPostAPI("/renter/upload/test2", uploadValues); err != nil { t.Fatal(err) } // Try renaming to a name that's already taken. renameValues := url.Values{} renameValues.Set("newsiapath", "test1") err = st.stdPostAPI("/renter/rename/test2", renameValues) if err == nil || err.Error() != renter.ErrPathOverload.Error() { t.Errorf("expected error to be %v; got %v", renter.ErrPathOverload, err) } // Try renaming to an empty string. renameValues.Set("newsiapath", "") err = st.stdPostAPI("/renter/rename/test1", renameValues) if err == nil || err.Error() != renter.ErrEmptyFilename.Error() { t.Fatalf("expected error to be %v; got %v", renter.ErrEmptyFilename, err) } // Rename the first file. renameValues.Set("newsiapath", "newtest1") if err = st.stdPostAPI("/renter/rename/test1", renameValues); err != nil { t.Fatal(err) } // Try renaming a nonexistent file. renameValues.Set("newsiapath", "newdne") err = st.stdPostAPI("/renter/rename/dne", renameValues) if err == nil || err.Error() != renter.ErrUnknownPath.Error() { t.Errorf("expected error to be %v; got %v", renter.ErrUnknownPath, err) } } // TestRenterHandlerDelete checks that deleting a valid file from the renter // goes as planned and that attempting to delete a nonexistent file fails with // the appropriate error. func TestRenterHandlerDelete(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRenterHandlerDelete") if err != nil { t.Fatal(err) } defer st.server.Close() // Anounce the host and start accepting contracts. if err := st.announceHost(); err != nil { t.Fatal(err) } if err = st.acceptContracts(); err != nil { t.Fatal(err) } if err = st.setHostStorage(); err != nil { t.Fatal(err) } // Set an allowance for the renter, allowing a contract to be formed. allowanceValues := url.Values{} allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", testPeriod) if err = st.stdPostAPI("/renter", allowanceValues); err != nil { t.Fatal(err) } // Create a file. path := filepath.Join(st.dir, "test.dat") if err = createRandFile(path, 1024); err != nil { t.Fatal(err) } // Upload to host. uploadValues := url.Values{} uploadValues.Set("source", path) if err = st.stdPostAPI("/renter/upload/test", uploadValues); err != nil { t.Fatal(err) } // Delete the file. if err = st.stdPostAPI("/renter/delete/test", url.Values{}); err != nil { t.Fatal(err) } // The renter's list of files should now be empty. var files RenterFiles if err = st.getAPI("/renter/files", &files); err != nil { t.Fatal(err) } if len(files.Files) != 0 { t.Fatalf("renter's list of files should be empty; got %v instead", files) } // Try deleting a nonexistent file. err = st.stdPostAPI("/renter/delete/dne", url.Values{}) if err == nil || err.Error() != renter.ErrUnknownPath.Error() { t.Errorf("expected error to be %v, got %v", renter.ErrUnknownPath, err) } } // Tests that the /renter/upload call checks for relative paths. func TestRenterRelativePathErrorUpload(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRenterRelativePathErrorUpload") if err != nil { t.Fatal(err) } defer st.server.Close() // Anounce the host and start accepting contracts. if err := st.announceHost(); err != nil { t.Fatal(err) } if err = st.acceptContracts(); err != nil { t.Fatal(err) } if err = st.setHostStorage(); err != nil { t.Fatal(err) } // Set an allowance for the renter, allowing a contract to be formed. allowanceValues := url.Values{} allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", testPeriod) if err = st.stdPostAPI("/renter", allowanceValues); err != nil { t.Fatal(err) } renterUploadAbsoluteError := "source must be an absolute path" // Create a file. path := filepath.Join(st.dir, "test.dat") if err = createRandFile(path, 1024); err != nil { t.Fatal(err) } // This should fail. uploadValues := url.Values{} uploadValues.Set("source", "test.dat") if err = st.stdPostAPI("/renter/upload/test", uploadValues); err.Error() != renterUploadAbsoluteError { t.Fatal(err) } // As should this. uploadValues = url.Values{} uploadValues.Set("source", "../test.dat") if err = st.stdPostAPI("/renter/upload/test", uploadValues); err.Error() != renterUploadAbsoluteError { t.Fatal(err) } // This should succeed. uploadValues = url.Values{} uploadValues.Set("source", path) if err = st.stdPostAPI("/renter/upload/test", uploadValues); err != nil { t.Fatal(err) } } // Tests that the /renter/download call checks for relative paths. func TestRenterRelativePathErrorDownload(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestRenterRelativePathErrorUpload") if err != nil { t.Fatal(err) } defer st.server.Close() // Anounce the host and start accepting contracts. if err := st.announceHost(); err != nil { t.Fatal(err) } if err = st.acceptContracts(); err != nil { t.Fatal(err) } if err = st.setHostStorage(); err != nil { t.Fatal(err) } // Set an allowance for the renter, allowing a contract to be formed. allowanceValues := url.Values{} allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", testPeriod) if err = st.stdPostAPI("/renter", allowanceValues); err != nil { t.Fatal(err) } renterDownloadAbsoluteError := "destination must be an absolute path" // Create a file. path := filepath.Join(st.dir, "test.dat") if err = createRandFile(path, 1024); err != nil { t.Fatal(err) } uploadValues := url.Values{} uploadValues.Set("source", path) if err = st.stdPostAPI("/renter/upload/test", uploadValues); err != nil { t.Fatal(err) } // This should fail. downloadPath := "test1.dat" if err = st.stdGetAPI("/renter/download/test?destination=" + downloadPath); err.Error() != renterDownloadAbsoluteError { t.Fatal(err) } // This should fail. downloadPath = "../test1.dat" if err = st.stdGetAPI("/renter/download/test?destination=" + downloadPath); err.Error() != renterDownloadAbsoluteError { t.Fatal(err) } // This should fail. downloadPath = filepath.Join(st.dir, "test1.dat") if err = st.stdGetAPI("/renter/download/test?destination=" + downloadPath); !strings.Contains(err.Error(), "contract") { t.Fatal(err) } } Sia-1.0.4/api/renterhost_test.go000066400000000000000000000162351302033070500165640ustar00rootroot00000000000000package api // renterhost_test.go sets up larger integration tests between renters and // hosts, checking that the whole storage ecosystem is functioning cohesively. // TODO: There are a bunch of magic numbers in this file. import ( "bytes" "io/ioutil" "net/url" "path/filepath" "testing" "time" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // TestIntegrationHostAndRent sets up an integration test where a host and // renter participate in all of the actions related to simple renting and // hosting. func TestIntegrationHostAndRent(t *testing.T) { t.Skip("failing due to contractor changes") if testing.Short() { t.SkipNow() } st, err := createServerTester("TestIntegrationHostAndRent") if err != nil { t.Fatal(err) } defer st.server.Close() // Announce the host and start accepting contracts. err = st.announceHost() if err != nil { t.Fatal(err) } err = st.acceptContracts() if err != nil { t.Fatal(err) } err = st.setHostStorage() if err != nil { t.Fatal(err) } // Set an allowance for the renter, allowing a contract to be formed. allowanceValues := url.Values{} testFunds := "10000000000000000000000000000" // 10k SC testPeriod := "5" allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", testPeriod) err = st.stdPostAPI("/renter", allowanceValues) if err != nil { t.Fatal(err) } // Create a file. path := filepath.Join(st.dir, "test.dat") err = createRandFile(path, 1024) if err != nil { t.Fatal(err) } // Upload to host. uploadValues := url.Values{} uploadValues.Set("source", path) err = st.stdPostAPI("/renter/upload/test", uploadValues) if err != nil { t.Fatal(err) } // Only one piece will be uploaded (10% at current redundancy). var rf RenterFiles for i := 0; i < 200 && (len(rf.Files) != 1 || rf.Files[0].UploadProgress < 10); i++ { st.getAPI("/renter/files", &rf) time.Sleep(100 * time.Millisecond) } if len(rf.Files) != 1 || rf.Files[0].UploadProgress < 10 { t.Fatal("the uploading is not succeeding for some reason:", rf.Files[0]) } // On a second connection, upload another file. path2 := filepath.Join(st.dir, "test2.dat") test2Size := modules.SectorSize*2 + 1 err = createRandFile(path2, int(test2Size)) if err != nil { t.Fatal(err) } uploadValues = url.Values{} uploadValues.Set("source", path2) err = st.stdPostAPI("/renter/upload/test2", uploadValues) if err != nil { t.Fatal(err) } // Only one piece will be uploaded (10% at current redundancy). for i := 0; i < 200 && (len(rf.Files) != 2 || rf.Files[0].UploadProgress < 10 || rf.Files[1].UploadProgress < 10); i++ { st.getAPI("/renter/files", &rf) time.Sleep(100 * time.Millisecond) } if len(rf.Files) != 2 || rf.Files[0].UploadProgress < 10 || rf.Files[1].UploadProgress < 10 { t.Fatal("the uploading is not succeeding for some reason:", rf.Files[0], rf.Files[1]) } // Try downloading the second file. downpath := filepath.Join(st.dir, "testdown.dat") err = st.stdGetAPI("/renter/download/test2?destination=" + downpath) if err != nil { t.Fatal(err) } // Check that the download has the right contents. orig, err := ioutil.ReadFile(path2) if err != nil { t.Fatal(err) } download, err := ioutil.ReadFile(downpath) if err != nil { t.Fatal(err) } if bytes.Compare(orig, download) != 0 { t.Fatal("data mismatch when downloading a file") } // The renter's downloads queue should have 1 entry now. var queue RenterDownloadQueue if err = st.getAPI("/renter/downloads", &queue); err != nil { t.Fatal(err) } if len(queue.Downloads) != 1 { t.Fatalf("expected renter to have 1 download in the queue; got %v", len(queue.Downloads)) } // Mine blocks until the host recognizes profit. The host will wait for 12 // blocks after the storage window has closed to report the profit, a total // of 40 blocks should be mined. for i := 0; i < 40; i++ { st.miner.AddBlock() } // Check that the host is reporting a profit. var hg HostGET st.getAPI("/host", &hg) if hg.FinancialMetrics.StorageRevenue.Cmp(types.ZeroCurrency) <= 0 || hg.FinancialMetrics.DownloadBandwidthRevenue.Cmp(types.ZeroCurrency) <= 0 { t.Log("Storage Revenue:", hg.FinancialMetrics.StorageRevenue) t.Log("Bandwidth Revenue:", hg.FinancialMetrics.DownloadBandwidthRevenue) t.Log("Full Financial Metrics:", hg.FinancialMetrics) t.Fatal("Host is not displaying revenue after resolving a storage proof.") } } // TestIntegrationUploadDownload tests that downloading and uploading in // parallel does not result in failures or stalling. func TestIntegrationUploadDownload(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestIntegrationUploadDownload") if err != nil { t.Fatal(err) } defer st.server.Close() // Announce the host and start accepting contracts. err = st.announceHost() if err != nil { t.Fatal(err) } err = st.acceptContracts() if err != nil { t.Fatal(err) } err = st.setHostStorage() if err != nil { t.Fatal(err) } // Set an allowance for the renter, allowing a contract to be formed. allowanceValues := url.Values{} testFunds := "10000000000000000000000000000" // 10k SC testPeriod := "5" allowanceValues.Set("funds", testFunds) allowanceValues.Set("period", testPeriod) err = st.stdPostAPI("/renter", allowanceValues) if err != nil { t.Fatal(err) } // Create a file. path := filepath.Join(st.dir, "test.dat") err = createRandFile(path, 1024) if err != nil { t.Fatal(err) } // Upload to host. uploadValues := url.Values{} uploadValues.Set("source", path) err = st.stdPostAPI("/renter/upload/test", uploadValues) if err != nil { t.Fatal(err) } // Only one piece will be uploaded (10% at current redundancy). var rf RenterFiles for i := 0; i < 200 && (len(rf.Files) != 1 || rf.Files[0].UploadProgress < 10); i++ { st.getAPI("/renter/files", &rf) time.Sleep(100 * time.Millisecond) } if len(rf.Files) != 1 || rf.Files[0].UploadProgress < 10 { t.Fatal("the uploading is not succeeding for some reason:", rf.Files[0]) } // In parallel, upload another file and download the first file. path2 := filepath.Join(st.dir, "test2.dat") test2Size := modules.SectorSize*2 + 1 err = createRandFile(path2, int(test2Size)) if err != nil { t.Fatal(err) } uploadValues = url.Values{} uploadValues.Set("source", path2) err = st.stdPostAPI("/renter/upload/test2", uploadValues) if err != nil { t.Fatal(err) } downpath := filepath.Join(st.dir, "testdown.dat") err = st.stdGetAPI("/renter/download/test?destination=" + downpath) if err != nil { t.Fatal(err) } // Check that the download has the right contents. orig, err := ioutil.ReadFile(path) if err != nil { t.Fatal(err) } download, err := ioutil.ReadFile(downpath) if err != nil { t.Fatal(err) } if bytes.Compare(orig, download) != 0 { t.Fatal("data mismatch when downloading a file") } // Wait for upload to complete. for i := 0; i < 200 && (len(rf.Files) != 2 || rf.Files[0].UploadProgress < 10 || rf.Files[1].UploadProgress < 10); i++ { st.getAPI("/renter/files", &rf) time.Sleep(100 * time.Millisecond) } if len(rf.Files) != 2 || rf.Files[0].UploadProgress < 10 || rf.Files[1].UploadProgress < 10 { t.Fatal("the uploading is not succeeding for some reason:", rf.Files[0], rf.Files[1]) } } Sia-1.0.4/api/scan.go000066400000000000000000000015621302033070500142510ustar00rootroot00000000000000package api import ( "math/big" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/types" ) // scanAmount scans a types.Currency from a string. func scanAmount(amount string) (types.Currency, bool) { // use SetString manually to ensure that amount does not contain // multiple values, which would confuse fmt.Scan i, ok := new(big.Int).SetString(amount, 10) if !ok { return types.Currency{}, ok } return types.NewCurrency(i), true } // scanAddress scans a types.UnlockHash from a string. func scanAddress(addrStr string) (addr types.UnlockHash, err error) { err = addr.LoadString(addrStr) if err != nil { return types.UnlockHash{}, err } return addr, nil } // scanHash scans a crypto.Hash from a string. func scanHash(s string) (h crypto.Hash, err error) { err = h.LoadString(s) if err != nil { return crypto.Hash{}, err } return h, nil } Sia-1.0.4/api/server_helpers_test.go000066400000000000000000000424551302033070500174220ustar00rootroot00000000000000package api import ( "encoding/json" "errors" "fmt" "io" "net" "net/http" "net/url" "path/filepath" "strings" "sync" "testing" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/consensus" "github.com/NebulousLabs/Sia/modules/explorer" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/modules/host" "github.com/NebulousLabs/Sia/modules/miner" "github.com/NebulousLabs/Sia/modules/renter" "github.com/NebulousLabs/Sia/modules/transactionpool" "github.com/NebulousLabs/Sia/modules/wallet" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/types" ) // A Server is essentially a collection of modules and an API server to talk // to them all. type Server struct { api *API apiServer *http.Server listener net.Listener requiredUserAgent string // wg is used to block Close() from returning until Serve() has finished. A // WaitGroup is used instead of a chan struct{} so that Close() can be called // without necessarily calling Serve() first. wg sync.WaitGroup } // NewServer creates a new API server from the provided modules. The API will // require authentication using HTTP basic auth if the supplied password is not // the empty string. Usernames are ignored for authentication. This type of // authentication sends passwords in plaintext and should therefore only be // used if the APIaddr is localhost. func NewServer(APIaddr string, requiredUserAgent string, requiredPassword string, cs modules.ConsensusSet, e modules.Explorer, g modules.Gateway, h modules.Host, m modules.Miner, r modules.Renter, tp modules.TransactionPool, w modules.Wallet) (*Server, error) { l, err := net.Listen("tcp", APIaddr) if err != nil { return nil, err } a := New(requiredUserAgent, requiredPassword, cs, e, g, h, m, r, tp, w) srv := &Server{ api: a, listener: l, requiredUserAgent: requiredUserAgent, apiServer: &http.Server{ Handler: a, }, } return srv, nil } // Serve listens for and handles API calls. It is a blocking function. func (srv *Server) Serve() error { // Block the Close() method until Serve() has finished. srv.wg.Add(1) defer srv.wg.Done() // The server will run until an error is encountered or the listener is // closed, via either the Close method or the signal handling above. // Closing the listener will result in the benign error handled below. err := srv.apiServer.Serve(srv.listener) if err != nil && !strings.HasSuffix(err.Error(), "use of closed network connection") { return err } return nil } // Close closes the Server's listener, causing the HTTP server to shut down. func (srv *Server) Close() error { var errs []error // Close the listener, which will cause Server.Serve() to return. if err := srv.listener.Close(); err != nil { errs = append(errs, fmt.Errorf("listener.Close failed: %v", err)) } // Wait for Server.Serve() to exit. We wait so that it's guaranteed that the // server has completely closed after Close() returns. This is particularly // useful during testing so that we don't exit a test before Serve() finishes. srv.wg.Wait() // Safely close each module. mods := []struct { name string c io.Closer }{ {"explorer", srv.api.explorer}, {"host", srv.api.host}, {"renter", srv.api.renter}, {"miner", srv.api.miner}, {"wallet", srv.api.wallet}, {"tpool", srv.api.tpool}, {"consensus", srv.api.cs}, {"gateway", srv.api.gateway}, } for _, mod := range mods { if mod.c != nil { if err := mod.c.Close(); err != nil { errs = append(errs, fmt.Errorf("%v.Close failed: %v", mod.name, err)) } } } return build.JoinErrors(errs, "\n") } // serverTester contains a server and a set of channels for keeping all of the // modules synchronized during testing. type serverTester struct { cs modules.ConsensusSet gateway modules.Gateway host modules.Host miner modules.TestMiner renter modules.Renter tpool modules.TransactionPool explorer modules.Explorer wallet modules.Wallet walletKey crypto.TwofishKey server *Server dir string } // assembleServerTester creates a bunch of modules and assembles them into a // server tester, without creating any directories or mining any blocks. func assembleServerTester(key crypto.TwofishKey, testdir string) (*serverTester, error) { // assembleServerTester should not get called during short tests, as it // takes a long time to run. if testing.Short() { panic("assembleServerTester called during short tests") } // Create the modules. g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, err } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { return nil, err } w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } if !w.Encrypted() { _, err = w.Encrypt(key) if err != nil { return nil, err } } err = w.Unlock(key) if err != nil { return nil, err } m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.MinerDir)) if err != nil { return nil, err } h, err := host.New(cs, tp, w, "localhost:0", filepath.Join(testdir, modules.HostDir)) if err != nil { return nil, err } r, err := renter.New(cs, w, tp, filepath.Join(testdir, modules.RenterDir)) if err != nil { return nil, err } srv, err := NewServer("localhost:0", "Sia-Agent", "", cs, nil, g, h, m, r, tp, w) if err != nil { return nil, err } // Assemble the serverTester. st := &serverTester{ cs: cs, gateway: g, host: h, miner: m, renter: r, tpool: tp, wallet: w, walletKey: key, server: srv, dir: testdir, } // TODO: A more reasonable way of listening for server errors. go func() { listenErr := srv.Serve() if listenErr != nil { panic(listenErr) } }() return st, nil } // assembleAuthenticatedServerTester creates a bunch of modules and assembles // them into a server tester that requires authentication with the given // requiredPassword. No directories are created and no blocks are mined. func assembleAuthenticatedServerTester(requiredPassword string, key crypto.TwofishKey, testdir string) (*serverTester, error) { // assembleAuthenticatedServerTester should not get called during short // tests, as it takes a long time to run. if testing.Short() { panic("assembleServerTester called during short tests") } // Create the modules. g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, err } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { return nil, err } w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } if !w.Encrypted() { _, err = w.Encrypt(key) if err != nil { return nil, err } } err = w.Unlock(key) if err != nil { return nil, err } m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.MinerDir)) if err != nil { return nil, err } h, err := host.New(cs, tp, w, "localhost:0", filepath.Join(testdir, modules.HostDir)) if err != nil { return nil, err } r, err := renter.New(cs, w, tp, filepath.Join(testdir, modules.RenterDir)) if err != nil { return nil, err } srv, err := NewServer("localhost:0", "Sia-Agent", requiredPassword, cs, nil, g, h, m, r, tp, w) if err != nil { return nil, err } // Assemble the serverTester. st := &serverTester{ cs: cs, gateway: g, host: h, miner: m, renter: r, tpool: tp, wallet: w, walletKey: key, server: srv, dir: testdir, } // TODO: A more reasonable way of listening for server errors. go func() { listenErr := srv.Serve() if listenErr != nil { panic(listenErr) } }() return st, nil } // assembleExplorerServerTester creates all the explorer dependencies and // explorer module without creating any directories. The user agent requirement // is disabled. func assembleExplorerServerTester(testdir string) (*serverTester, error) { // assembleExplorerServerTester should not get called during short tests, // as it takes a long time to run. if testing.Short() { panic("assembleServerTester called during short tests") } // Create the modules. g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, err } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } e, err := explorer.New(cs, filepath.Join(testdir, modules.ExplorerDir)) if err != nil { return nil, err } srv, err := NewServer("localhost:0", "", "", cs, e, g, nil, nil, nil, nil, nil) if err != nil { return nil, err } // Assemble the serverTester. st := &serverTester{ cs: cs, explorer: e, gateway: g, server: srv, dir: testdir, } // TODO: A more reasonable way of listening for server errors. go func() { listenErr := srv.Serve() if listenErr != nil { panic(listenErr) } }() return st, nil } // blankServerTester creates a server tester object that is ready for testing, // without any money in the wallet. func blankServerTester(name string) (*serverTester, error) { // createServerTester is expensive, and therefore should not be called // during short tests. if testing.Short() { panic("blankServerTester called during short tests") } // Create the server tester with key. testdir := build.TempDir("api", name) key, err := crypto.GenerateTwofishKey() if err != nil { return nil, err } st, err := assembleServerTester(key, testdir) if err != nil { return nil, err } return st, nil } // createServerTester creates a server tester object that is ready for testing, // including money in the wallet and all modules initialized. func createServerTester(name string) (*serverTester, error) { // createServerTester is expensive, and therefore should not be called // during short tests. if testing.Short() { panic("createServerTester called during short tests") } // Create the testing directory. testdir := build.TempDir("api", name) key, err := crypto.GenerateTwofishKey() if err != nil { return nil, err } st, err := assembleServerTester(key, testdir) if err != nil { return nil, err } // Mine blocks until the wallet has confirmed money. for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ { _, err := st.miner.AddBlock() if err != nil { return nil, err } } return st, nil } // createAuthenticatedServerTester creates an authenticated server tester // object that is ready for testing, including money in the wallet and all // modules initalized. func createAuthenticatedServerTester(name string, password string) (*serverTester, error) { // createAuthenticatedServerTester should not get called during short // tests, as it takes a long time to run. if testing.Short() { panic("assembleServerTester called during short tests") } // Create the testing directory. testdir := build.TempDir("authenticated-api", name) key, err := crypto.GenerateTwofishKey() if err != nil { return nil, err } st, err := assembleAuthenticatedServerTester(password, key, testdir) if err != nil { return nil, err } // Mine blocks until the wallet has confirmed money. for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ { _, err := st.miner.AddBlock() if err != nil { return nil, err } } return st, nil } // createExplorerServerTester creates a server tester object containing only // the explorer and some presets that match standard explorer setups. func createExplorerServerTester(name string) (*serverTester, error) { testdir := build.TempDir("api", name) st, err := assembleExplorerServerTester(testdir) if err != nil { return nil, err } return st, nil } // non2xx returns true for non-success HTTP status codes. func non2xx(code int) bool { return code < 200 || code > 299 } // decodeError returns the api.Error from a API response. This method should // only be called if the response's status code is non-2xx. The error returned // may not be of type api.Error in the event of an error unmarshalling the // JSON. func decodeError(resp *http.Response) error { var apiErr Error err := json.NewDecoder(resp.Body).Decode(&apiErr) if err != nil { return err } return apiErr } // reloadedServerTester creates a server tester where all of the persistent // data has been copied to a new folder and all of the modules re-initialized // on the new folder. This gives an opportunity to see how modules will behave // when they are relying on their persistent structures. func (st *serverTester) reloadedServerTester() (*serverTester, error) { // Copy the testing directory. copiedDir := st.dir + " - " + persist.RandomSuffix() err := build.CopyDir(st.dir, copiedDir) if err != nil { return nil, err } copyST, err := assembleServerTester(st.walletKey, copiedDir) if err != nil { return nil, err } return copyST, nil } // netAddress returns the NetAddress of the caller. func (st *serverTester) netAddress() modules.NetAddress { return st.server.api.gateway.Address() } // coinAddress returns a coin address that the caller is able to spend from. func (st *serverTester) coinAddress() string { var addr struct { Address string } st.getAPI("/wallet/address", &addr) return addr.Address } // acceptContracts instructs the host to begin accepting contracts. func (st *serverTester) acceptContracts() error { settingsValues := url.Values{} settingsValues.Set("acceptingcontracts", "true") return st.stdPostAPI("/host", settingsValues) } // setHostStorage adds a 1 GB folder to the host. func (st *serverTester) setHostStorage() error { values := url.Values{} values.Set("path", st.dir) values.Set("size", "1048576") return st.stdPostAPI("/host/storage/folders/add", values) } // announceHost announces the host, mines a block, and waits for the // announcement to register. func (st *serverTester) announceHost() error { // Set the host to be accepting contracts. acceptingContractsValues := url.Values{} acceptingContractsValues.Set("acceptingcontracts", "true") err := st.stdPostAPI("/host", acceptingContractsValues) if err != nil { return err } announceValues := url.Values{} announceValues.Set("address", string(st.host.ExternalSettings().NetAddress)) err = st.stdPostAPI("/host/announce", announceValues) if err != nil { return err } // mine block _, err = st.miner.AddBlock() if err != nil { return err } // wait for announcement var hosts ActiveHosts err = st.getAPI("/hostdb/active", &hosts) if err != nil { return err } for i := 0; i < 50 && len(hosts.Hosts) == 0; i++ { time.Sleep(100 * time.Millisecond) err = st.getAPI("/hostdb/active", &hosts) if err != nil { return err } } if len(hosts.Hosts) == 0 { return errors.New("host announcement not seen") } return nil } // getAPI makes an API call and decodes the response. func (st *serverTester) getAPI(call string, obj interface{}) error { resp, err := HttpGET("http://" + st.server.listener.Addr().String() + call) if err != nil { return err } defer resp.Body.Close() if non2xx(resp.StatusCode) { return decodeError(resp) } // Return early because there is no content to decode. if resp.StatusCode == http.StatusNoContent { return nil } // Decode the response into 'obj'. err = json.NewDecoder(resp.Body).Decode(obj) if err != nil { return err } return nil } // postAPI makes an API call and decodes the response. func (st *serverTester) postAPI(call string, values url.Values, obj interface{}) error { resp, err := HttpPOST("http://"+st.server.listener.Addr().String()+call, values.Encode()) if err != nil { return err } defer resp.Body.Close() if non2xx(resp.StatusCode) { return decodeError(resp) } // Return early because there is no content to decode. if resp.StatusCode == http.StatusNoContent { return nil } // Decode the response into 'obj'. err = json.NewDecoder(resp.Body).Decode(obj) if err != nil { return err } return nil } // stdGetAPI makes an API call and discards the response. func (st *serverTester) stdGetAPI(call string) error { resp, err := HttpGET("http://" + st.server.listener.Addr().String() + call) if err != nil { return err } defer resp.Body.Close() if non2xx(resp.StatusCode) { return decodeError(resp) } return nil } // stdGetAPIUA makes an API call with a custom user agent. func (st *serverTester) stdGetAPIUA(call string, userAgent string) error { req, err := http.NewRequest("GET", "http://"+st.server.listener.Addr().String()+call, nil) if err != nil { return err } req.Header.Set("User-Agent", userAgent) resp, err := http.DefaultClient.Do(req) if err != nil { return err } defer resp.Body.Close() if non2xx(resp.StatusCode) { return decodeError(resp) } return nil } // stdPostAPI makes an API call and discards the response. func (st *serverTester) stdPostAPI(call string, values url.Values) error { resp, err := HttpPOST("http://"+st.server.listener.Addr().String()+call, values.Encode()) if err != nil { return err } defer resp.Body.Close() if non2xx(resp.StatusCode) { return decodeError(resp) } return nil } Sia-1.0.4/api/server_test.go000066400000000000000000000067271302033070500157020ustar00rootroot00000000000000package api import ( "net/http" "testing" ) // TestExplorerPreset checks that the default configuration for the explorer is // working correctly. func TestExplorerPreset(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createExplorerServerTester("TestExplorerPreset") if err != nil { t.Fatal(err) } defer st.server.Close() // Try calling a legal endpoint without a user agent. err = st.stdGetAPIUA("/explorer", "") if err != nil { t.Fatal(err) } } // TestReloading reloads a server and does smoke testing to see that modules // are still working after reload. func TestReloading(t *testing.T) { if testing.Short() { t.SkipNow() } // Create a server tester, which will have blocks mined. Then get the // reloaded version of the server tester (all persistence files get copied // to a new folder, and then the modules are pointed at the new folders // during calls to 'New') st, err := createServerTester("TestReloading") height := st.server.api.cs.Height() if err != nil { t.Fatal(err) } err = st.server.Close() if err != nil { t.Fatal(err) } rst, err := st.reloadedServerTester() if err != nil { t.Fatal(err) } defer rst.server.Close() if height != rst.server.api.cs.Height() { t.Error("server heights do not match") } // Mine some blocks on the reloaded server and see if any errors or panics // are triggered. for i := 0; i < 3; i++ { _, err := rst.miner.AddBlock() if err != nil { t.Fatal(err) } } } // TestAuthenticated tests creating a server that requires authenticated API // calls, and then makes (un)authenticated API calls to test the // authentication. func TestAuthentication(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createAuthenticatedServerTester("TestAuthentication", "password") if err != nil { t.Fatal(err) } defer st.server.Close() testGETURL := "http://" + st.server.listener.Addr().String() + "/wallet/seeds" testPOSTURL := "http://" + st.server.listener.Addr().String() + "/host/announce" // Test that unauthenticated API calls fail. // GET resp, err := HttpGET(testGETURL) if err != nil { t.Fatal(err) } if resp.StatusCode != http.StatusUnauthorized { t.Fatal("unauthenticated API call succeeded on a server that requires authentication") } // POST resp, err = HttpPOST(testPOSTURL, "") if err != nil { t.Fatal(err) } if resp.StatusCode != http.StatusUnauthorized { t.Fatal("unauthenticated API call succeeded on a server that requires authentication") } // Test that authenticated API calls with the wrong password fail. // GET resp, err = HttpGETAuthenticated(testGETURL, "wrong password") if err != nil { t.Fatal(err) } if resp.StatusCode != http.StatusUnauthorized { t.Fatal("authenticated API call succeeded with an incorrect password") } // POST resp, err = HttpPOSTAuthenticated(testPOSTURL, "", "wrong password") if err != nil { t.Fatal(err) } if resp.StatusCode != http.StatusUnauthorized { t.Fatal("authenticated API call succeeded with an incorrect password") } // Test that authenticated API calls with the correct password succeed. // GET resp, err = HttpGETAuthenticated(testGETURL, "password") if err != nil { t.Fatal(err) } if non2xx(resp.StatusCode) { t.Fatal("authenticated API call failed with the correct password") } // POST resp, err = HttpPOSTAuthenticated(testPOSTURL, "", "password") if err != nil { t.Fatal(err) } if non2xx(resp.StatusCode) { t.Fatal("authenticated API call failed with the correct password") } } Sia-1.0.4/api/transactionpool.go000066400000000000000000000007711302033070500165450ustar00rootroot00000000000000package api import ( "net/http" "github.com/NebulousLabs/Sia/types" "github.com/julienschmidt/httprouter" ) type TransactionPoolGET struct { Transactions []types.Transaction `json:"transactions"` } // transactionpoolTransactionsHandler handles the API call to get the // transaction pool trasactions. func (api *API) transactionpoolTransactionsHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { WriteJSON(w, TransactionPoolGET{Transactions: api.tpool.TransactionList()}) } Sia-1.0.4/api/wallet.go000066400000000000000000000367441302033070500146270ustar00rootroot00000000000000package api import ( "net/http" "path/filepath" "strconv" "strings" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/entropy-mnemonics" "github.com/julienschmidt/httprouter" ) type ( // WalletGET contains general information about the wallet. WalletGET struct { Encrypted bool `json:"encrypted"` Unlocked bool `json:"unlocked"` ConfirmedSiacoinBalance types.Currency `json:"confirmedsiacoinbalance"` UnconfirmedOutgoingSiacoins types.Currency `json:"unconfirmedoutgoingsiacoins"` UnconfirmedIncomingSiacoins types.Currency `json:"unconfirmedincomingsiacoins"` SiafundBalance types.Currency `json:"siafundbalance"` SiacoinClaimBalance types.Currency `json:"siacoinclaimbalance"` } // WalletAddressGET contains an address returned by a GET call to // /wallet/address. WalletAddressGET struct { Address types.UnlockHash `json:"address"` } // WalletAddressesGET contains the list of wallet addresses returned by a // GET call to /wallet/addresses. WalletAddressesGET struct { Addresses []types.UnlockHash `json:"addresses"` } // WalletInitPOST contains the primary seed that gets generated during a // POST call to /wallet/init. WalletInitPOST struct { PrimarySeed string `json:"primaryseed"` } // WalletSiacoinsPOST contains the transaction sent in the POST call to // /wallet/siafunds. WalletSiacoinsPOST struct { TransactionIDs []types.TransactionID `json:"transactionids"` } // WalletSiafundsPOST contains the transaction sent in the POST call to // /wallet/siafunds. WalletSiafundsPOST struct { TransactionIDs []types.TransactionID `json:"transactionids"` } // WalletSeedsGET contains the seeds used by the wallet. WalletSeedsGET struct { PrimarySeed string `json:"primaryseed"` AddressesRemaining int `json:"addressesremaining"` AllSeeds []string `json:"allseeds"` } // WalletTransactionGETid contains the transaction returned by a call to // /wallet/transaction/$(id) WalletTransactionGETid struct { Transaction modules.ProcessedTransaction `json:"transaction"` } // WalletTransactionsGET contains the specified set of confirmed and // unconfirmed transactions. WalletTransactionsGET struct { ConfirmedTransactions []modules.ProcessedTransaction `json:"confirmedtransactions"` UnconfirmedTransactions []modules.ProcessedTransaction `json:"unconfirmedtransactions"` } // WalletTransactionsGETaddr contains the set of wallet transactions // relevant to the input address provided in the call to // /wallet/transaction/$(addr) WalletTransactionsGETaddr struct { ConfirmedTransactions []modules.ProcessedTransaction `json:"confirmedtransactions"` UnconfirmedTransactions []modules.ProcessedTransaction `json:"unconfirmedtransactions"` } ) // encryptionKeys enumerates the possible encryption keys that can be derived // from an input string. func encryptionKeys(seedStr string) (validKeys []crypto.TwofishKey) { dicts := []mnemonics.DictionaryID{"english", "german", "japanese"} for _, dict := range dicts { seed, err := modules.StringToSeed(seedStr, dict) if err != nil { continue } validKeys = append(validKeys, crypto.TwofishKey(crypto.HashObject(seed))) } validKeys = append(validKeys, crypto.TwofishKey(crypto.HashObject(seedStr))) return validKeys } // walletHander handles API calls to /wallet. func (api *API) walletHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { siacoinBal, siafundBal, siaclaimBal := api.wallet.ConfirmedBalance() siacoinsOut, siacoinsIn := api.wallet.UnconfirmedBalance() WriteJSON(w, WalletGET{ Encrypted: api.wallet.Encrypted(), Unlocked: api.wallet.Unlocked(), ConfirmedSiacoinBalance: siacoinBal, UnconfirmedOutgoingSiacoins: siacoinsOut, UnconfirmedIncomingSiacoins: siacoinsIn, SiafundBalance: siafundBal, SiacoinClaimBalance: siaclaimBal, }) } // wallet033xHandler handles API calls to /wallet/033x. func (api *API) wallet033xHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { source := req.FormValue("source") // Check that source is an absolute paths. if !filepath.IsAbs(source) { WriteError(w, Error{"error when calling /wallet/033x: source must be an absolute path"}, http.StatusBadRequest) return } potentialKeys := encryptionKeys(req.FormValue("encryptionpassword")) for _, key := range potentialKeys { err := api.wallet.Load033xWallet(key, source) if err == nil { WriteSuccess(w) return } if err != nil && err != modules.ErrBadEncryptionKey { WriteError(w, Error{"error when calling /wallet/033x: " + err.Error()}, http.StatusBadRequest) return } } WriteError(w, Error{modules.ErrBadEncryptionKey.Error()}, http.StatusBadRequest) } // walletAddressHandler handles API calls to /wallet/address. func (api *API) walletAddressHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { unlockConditions, err := api.wallet.NextAddress() if err != nil { WriteError(w, Error{"error after call to /wallet/addresses: " + err.Error()}, http.StatusBadRequest) return } WriteJSON(w, WalletAddressGET{ Address: unlockConditions.UnlockHash(), }) } // walletAddressHandler handles API calls to /wallet/addresses. func (api *API) walletAddressesHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { WriteJSON(w, WalletAddressesGET{ Addresses: api.wallet.AllAddresses(), }) } // walletBackupHandler handles API calls to /wallet/backup. func (api *API) walletBackupHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { destination := req.FormValue("destination") // Check that the destination is absolute. if !filepath.IsAbs(destination) { WriteError(w, Error{"error when calling /wallet/backup: destination must be an absolute path"}, http.StatusBadRequest) return } err := api.wallet.CreateBackup(destination) if err != nil { WriteError(w, Error{"error after call to /wallet/backup: " + err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } // walletInitHandler handles API calls to /wallet/init. func (api *API) walletInitHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { var encryptionKey crypto.TwofishKey if req.FormValue("encryptionpassword") != "" { encryptionKey = crypto.TwofishKey(crypto.HashObject(req.FormValue("encryptionpassword"))) } seed, err := api.wallet.Encrypt(encryptionKey) if err != nil { WriteError(w, Error{"error when calling /wallet/init: " + err.Error()}, http.StatusBadRequest) return } dictID := mnemonics.DictionaryID(req.FormValue("dictionary")) if dictID == "" { dictID = "english" } seedStr, err := modules.SeedToString(seed, dictID) if err != nil { WriteError(w, Error{"error when calling /wallet/init: " + err.Error()}, http.StatusBadRequest) return } WriteJSON(w, WalletInitPOST{ PrimarySeed: seedStr, }) } // walletSeedHandler handles API calls to /wallet/seed. func (api *API) walletSeedHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { // Get the seed using the ditionary + phrase dictID := mnemonics.DictionaryID(req.FormValue("dictionary")) if dictID == "" { dictID = "english" } seed, err := modules.StringToSeed(req.FormValue("seed"), dictID) if err != nil { WriteError(w, Error{"error when calling /wallet/seed: " + err.Error()}, http.StatusBadRequest) return } potentialKeys := encryptionKeys(req.FormValue("encryptionpassword")) for _, key := range potentialKeys { err := api.wallet.LoadSeed(key, seed) if err == nil { WriteSuccess(w) return } if err != nil && err != modules.ErrBadEncryptionKey { WriteError(w, Error{"error when calling /wallet/seed: " + err.Error()}, http.StatusBadRequest) return } } WriteError(w, Error{"error when calling /wallet/seed: " + modules.ErrBadEncryptionKey.Error()}, http.StatusBadRequest) } // walletSiagkeyHandler handles API calls to /wallet/siagkey. func (api *API) walletSiagkeyHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { // Fetch the list of keyfiles from the post body. keyfiles := strings.Split(req.FormValue("keyfiles"), ",") potentialKeys := encryptionKeys(req.FormValue("encryptionpassword")) for _, keypath := range keyfiles { // Check that all key paths are absolute paths. if !filepath.IsAbs(keypath) { WriteError(w, Error{"error when calling /wallet/siagkey: keyfiles contains a non-absolute path"}, http.StatusBadRequest) return } } for _, key := range potentialKeys { err := api.wallet.LoadSiagKeys(key, keyfiles) if err == nil { WriteSuccess(w) return } if err != nil && err != modules.ErrBadEncryptionKey { WriteError(w, Error{"error when calling /wallet/siagkey: " + err.Error()}, http.StatusBadRequest) return } } WriteError(w, Error{"error when calling /wallet/siagkey: " + modules.ErrBadEncryptionKey.Error()}, http.StatusBadRequest) } // walletLockHanlder handles API calls to /wallet/lock. func (api *API) walletLockHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { err := api.wallet.Lock() if err != nil { WriteError(w, Error{err.Error()}, http.StatusBadRequest) return } WriteSuccess(w) } // walletSeedsHandler handles API calls to /wallet/seeds. func (api *API) walletSeedsHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { dictionary := mnemonics.DictionaryID(req.FormValue("dictionary")) if dictionary == "" { dictionary = mnemonics.English } // Get the primary seed information. primarySeed, progress, err := api.wallet.PrimarySeed() if err != nil { WriteError(w, Error{"error after call to /wallet/seeds: " + err.Error()}, http.StatusBadRequest) return } primarySeedStr, err := modules.SeedToString(primarySeed, dictionary) if err != nil { WriteError(w, Error{"error after call to /wallet/seeds: " + err.Error()}, http.StatusBadRequest) return } // Get the list of seeds known to the wallet. allSeeds, err := api.wallet.AllSeeds() if err != nil { WriteError(w, Error{"error after call to /wallet/seeds: " + err.Error()}, http.StatusBadRequest) return } var allSeedsStrs []string for _, seed := range allSeeds { str, err := modules.SeedToString(seed, dictionary) if err != nil { WriteError(w, Error{"error after call to /wallet/seeds: " + err.Error()}, http.StatusBadRequest) return } allSeedsStrs = append(allSeedsStrs, str) } WriteJSON(w, WalletSeedsGET{ PrimarySeed: primarySeedStr, AddressesRemaining: int(modules.PublicKeysPerSeed - progress), AllSeeds: allSeedsStrs, }) } // walletSiacoinsHandler handles API calls to /wallet/siacoins. func (api *API) walletSiacoinsHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { amount, ok := scanAmount(req.FormValue("amount")) if !ok { WriteError(w, Error{"could not read 'amount' from POST call to /wallet/siacoins"}, http.StatusBadRequest) return } dest, err := scanAddress(req.FormValue("destination")) if err != nil { WriteError(w, Error{"error after call to /wallet/siacoins: " + err.Error()}, http.StatusBadRequest) return } txns, err := api.wallet.SendSiacoins(amount, dest) if err != nil { WriteError(w, Error{"error after call to /wallet/siacoins: " + err.Error()}, http.StatusInternalServerError) return } var txids []types.TransactionID for _, txn := range txns { txids = append(txids, txn.ID()) } WriteJSON(w, WalletSiacoinsPOST{ TransactionIDs: txids, }) } // walletSiafundsHandler handles API calls to /wallet/siafunds. func (api *API) walletSiafundsHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { amount, ok := scanAmount(req.FormValue("amount")) if !ok { WriteError(w, Error{"could not read 'amount' from POST call to /wallet/siafunds"}, http.StatusBadRequest) return } dest, err := scanAddress(req.FormValue("destination")) if err != nil { WriteError(w, Error{"error after call to /wallet/siafunds: " + err.Error()}, http.StatusBadRequest) return } txns, err := api.wallet.SendSiafunds(amount, dest) if err != nil { WriteError(w, Error{"error after call to /wallet/siafunds: " + err.Error()}, http.StatusInternalServerError) return } var txids []types.TransactionID for _, txn := range txns { txids = append(txids, txn.ID()) } WriteJSON(w, WalletSiafundsPOST{ TransactionIDs: txids, }) } // walletTransactionHandler handles API calls to /wallet/transaction/:id. func (api *API) walletTransactionHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // Parse the id from the url. var id types.TransactionID jsonID := "\"" + ps.ByName("id") + "\"" err := id.UnmarshalJSON([]byte(jsonID)) if err != nil { WriteError(w, Error{"error after call to /wallet/history: " + err.Error()}, http.StatusBadRequest) return } txn, ok := api.wallet.Transaction(id) if !ok { WriteError(w, Error{"error when calling /wallet/transaction/$(id): transaction not found"}, http.StatusBadRequest) return } WriteJSON(w, WalletTransactionGETid{ Transaction: txn, }) } // walletTransactionsHandler handles API calls to /wallet/transactions. func (api *API) walletTransactionsHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { startheightStr, endheightStr := req.FormValue("startheight"), req.FormValue("endheight") if startheightStr == "" || endheightStr == "" { WriteError(w, Error{"startheight and endheight must be provided to a /wallet/transactions call."}, http.StatusBadRequest) return } // Get the start and end blocks. start, err := strconv.Atoi(startheightStr) if err != nil { WriteError(w, Error{"parsing integer value for parameter `startheight` failed: " + err.Error()}, http.StatusBadRequest) return } end, err := strconv.Atoi(endheightStr) if err != nil { WriteError(w, Error{"parsing integer value for parameter `endheight` failed: " + err.Error()}, http.StatusBadRequest) return } confirmedTxns, err := api.wallet.Transactions(types.BlockHeight(start), types.BlockHeight(end)) if err != nil { WriteError(w, Error{"error after call to /wallet/transactions: " + err.Error()}, http.StatusBadRequest) return } unconfirmedTxns := api.wallet.UnconfirmedTransactions() WriteJSON(w, WalletTransactionsGET{ ConfirmedTransactions: confirmedTxns, UnconfirmedTransactions: unconfirmedTxns, }) } // walletTransactionsAddrHandler handles API calls to // /wallet/transactions/:addr. func (api *API) walletTransactionsAddrHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { // Parse the address being input. jsonAddr := "\"" + ps.ByName("addr") + "\"" var addr types.UnlockHash err := addr.UnmarshalJSON([]byte(jsonAddr)) if err != nil { WriteError(w, Error{"error after call to /wallet/transactions: " + err.Error()}, http.StatusBadRequest) return } confirmedATs := api.wallet.AddressTransactions(addr) unconfirmedATs := api.wallet.AddressUnconfirmedTransactions(addr) WriteJSON(w, WalletTransactionsGETaddr{ ConfirmedTransactions: confirmedATs, UnconfirmedTransactions: unconfirmedATs, }) } // walletUnlockHandler handles API calls to /wallet/unlock. func (api *API) walletUnlockHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { potentialKeys := encryptionKeys(req.FormValue("encryptionpassword")) for _, key := range potentialKeys { err := api.wallet.Unlock(key) if err == nil { WriteSuccess(w) return } if err != nil && err != modules.ErrBadEncryptionKey { WriteError(w, Error{"error when calling /wallet/unlock: " + err.Error()}, http.StatusBadRequest) return } } WriteError(w, Error{"error when calling /wallet/unlock: " + modules.ErrBadEncryptionKey.Error()}, http.StatusBadRequest) } Sia-1.0.4/api/wallet_test.go000066400000000000000000000357611302033070500156640ustar00rootroot00000000000000package api import ( "fmt" "net/url" "os" "path/filepath" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/consensus" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/modules/transactionpool" "github.com/NebulousLabs/Sia/modules/wallet" "github.com/NebulousLabs/Sia/types" ) // TestIntegrationWalletGETEncrypted probes the GET call to /wallet when the // wallet has never been encrypted. func TestIntegrationWalletGETEncrypted(t *testing.T) { if testing.Short() { t.SkipNow() } // Check a wallet that has never been encrypted. testdir := build.TempDir("api", "TestIntegrationWalletGETEncrypted") g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { t.Fatal("Failed to create gateway:", err) } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { t.Fatal("Failed to create consensus set:", err) } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { t.Fatal("Failed to create tpool:", err) } w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { t.Fatal("Failed to create wallet:", err) } srv, err := NewServer("localhost:0", "Sia-Agent", "", cs, nil, g, nil, nil, nil, tp, w) if err != nil { t.Fatal(err) } // Assemble the serverTester and start listening for api requests. st := &serverTester{ cs: cs, gateway: g, tpool: tp, wallet: w, server: srv, } errChan := make(chan error) go func() { listenErr := srv.Serve() errChan <- listenErr }() defer func() { err := <-errChan if err != nil { t.Fatalf("API server quit: %v", err) } }() defer st.server.Close() var wg WalletGET err = st.getAPI("/wallet", &wg) if err != nil { t.Fatal(err) } if wg.Encrypted { t.Error("Wallet has never been encrypted") } if wg.Unlocked { t.Error("Wallet has never been unlocked") } } // TestIntegrationWalletBlankEncrypt tries to encrypt and unlock the wallet // through the api using a blank encryption key - meaning that the wallet seed // returned by the encryption call can be used as the encryption key. func TestIntegrationWalletBlankEncrypt(t *testing.T) { if testing.Short() { t.SkipNow() } // Create a server object without encrypting or unlocking the wallet. testdir := build.TempDir("api", "TestIntegrationWalletBlankEncrypt") g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { t.Fatal(err) } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { t.Fatal(err) } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { t.Fatal(err) } w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { t.Fatal(err) } srv, err := NewServer("localhost:0", "Sia-Agent", "", cs, nil, g, nil, nil, nil, tp, w) if err != nil { t.Fatal(err) } // Assemble the serverTester. st := &serverTester{ cs: cs, gateway: g, tpool: tp, wallet: w, server: srv, } go func() { listenErr := srv.Serve() if listenErr != nil { panic(listenErr) } }() defer st.server.Close() // Make a call to /wallet/init and get the seed. Provide no encryption // key so that the encryption key is the seed that gets returned. var wip WalletInitPOST err = st.postAPI("/wallet/init", url.Values{}, &wip) if err != nil { t.Fatal(err) } // Use the seed to call /wallet/unlock. unlockValues := url.Values{} unlockValues.Set("encryptionpassword", wip.PrimarySeed) err = st.stdPostAPI("/wallet/unlock", unlockValues) if err != nil { t.Fatal(err) } // Check that the wallet actually unlocked. if !w.Unlocked() { t.Error("wallet is not unlocked") } } // TestIntegrationWalletGETSiacoins probes the GET call to /wallet when the // siacoin balance is being manipulated. func TestIntegrationWalletGETSiacoins(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestIntegrationWalletGETSiacoins") if err != nil { t.Fatal(err) } defer st.server.Close() // Check the initial wallet is encrypted, unlocked, and has the siacoins // that got mined. var wg WalletGET err = st.getAPI("/wallet", &wg) if err != nil { t.Fatal(err) } if !wg.Encrypted { t.Error("Wallet has been encrypted") } if !wg.Unlocked { t.Error("Wallet has been unlocked") } if wg.ConfirmedSiacoinBalance.Cmp(types.CalculateCoinbase(1)) != 0 { t.Error("reported wallet balance does not reflect the single block that has been mined") } if wg.UnconfirmedOutgoingSiacoins.Cmp64(0) != 0 { t.Error("there should not be unconfirmed outgoing siacoins") } if wg.UnconfirmedIncomingSiacoins.Cmp64(0) != 0 { t.Error("there should not be unconfirmed incoming siacoins") } // Send coins to a wallet address through the api. var wag WalletAddressGET err = st.getAPI("/wallet/address", &wag) if err != nil { t.Fatal(err) } sendSiacoinsValues := url.Values{} sendSiacoinsValues.Set("amount", "1234") sendSiacoinsValues.Set("destination", wag.Address.String()) err = st.stdPostAPI("/wallet/siacoins", sendSiacoinsValues) if err != nil { t.Fatal(err) } // Check that the wallet is reporting unconfirmed siacoins. err = st.getAPI("/wallet", &wg) if err != nil { t.Fatal(err) } if !wg.Encrypted { t.Error("Wallet has been encrypted") } if !wg.Unlocked { t.Error("Wallet has been unlocked") } if wg.ConfirmedSiacoinBalance.Cmp(types.CalculateCoinbase(1)) != 0 { t.Error("reported wallet balance does not reflect the single block that has been mined") } if wg.UnconfirmedOutgoingSiacoins.Cmp64(0) <= 0 { t.Error("there should be unconfirmed outgoing siacoins") } if wg.UnconfirmedIncomingSiacoins.Cmp64(0) <= 0 { t.Error("there should be unconfirmed incoming siacoins") } if wg.UnconfirmedOutgoingSiacoins.Cmp(wg.UnconfirmedIncomingSiacoins) <= 0 { t.Error("net movement of siacoins should be outgoing (miner fees)") } // Mine a block and see that the unconfirmed balances reduce back to // nothing. _, err = st.miner.AddBlock() if err != nil { t.Fatal(err) } err = st.getAPI("/wallet", &wg) if err != nil { t.Fatal(err) } if wg.ConfirmedSiacoinBalance.Cmp(types.CalculateCoinbase(1).Add(types.CalculateCoinbase(2))) >= 0 { t.Error("reported wallet balance does not reflect mining two blocks and eating a miner fee") } if wg.UnconfirmedOutgoingSiacoins.Cmp64(0) != 0 { t.Error("there should not be unconfirmed outgoing siacoins") } if wg.UnconfirmedIncomingSiacoins.Cmp64(0) != 0 { t.Error("there should not be unconfirmed incoming siacoins") } } // TestIntegrationWalletTransactionGETid queries the /wallet/transaction/$(id) // api call. func TestIntegrationWalletTransactionGETid(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestIntegrationWalletTransactionGETid") if err != nil { t.Fatal(err) } defer st.server.Close() // Mining blocks should have created transactions for the wallet containing // miner payouts. Get the list of transactions. var wtg WalletTransactionsGET err = st.getAPI("/wallet/transactions?startheight=0&endheight=10", &wtg) if err != nil { t.Fatal(err) } if len(wtg.ConfirmedTransactions) == 0 { t.Error("expecting a few wallet transactions, corresponding to miner payouts.") } if len(wtg.UnconfirmedTransactions) != 0 { t.Error("expecting 0 unconfirmed transactions") } // A call to /wallet/transactions without startheight and endheight parameters // should return a descriptive error message. err = st.getAPI("/wallet/transactions", &wtg) if err == nil || err.Error() != "startheight and endheight must be provided to a /wallet/transactions call." { t.Error("expecting /wallet/transactions call with empty parameters to error") } // Query the details of the first transaction using // /wallet/transaction/$(id) var wtgid WalletTransactionGETid wtgidQuery := fmt.Sprintf("/wallet/transaction/%s", wtg.ConfirmedTransactions[0].TransactionID) err = st.getAPI(wtgidQuery, &wtgid) if err != nil { t.Fatal(err) } if len(wtgid.Transaction.Inputs) != 0 { t.Error("miner payout should appear as an output, not an input") } if len(wtgid.Transaction.Outputs) != 1 { t.Fatal("a single miner payout output should have been created") } if wtgid.Transaction.Outputs[0].FundType != types.SpecifierMinerPayout { t.Error("fund type should be a miner payout") } } // Tests that the /wallet/backup call checks for relative paths. func TestWalletRelativePathErrorBackup(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestWalletRelativePathErrorBackup") if err != nil { t.Fatal(err) } defer st.server.Close() // Announce the host. if err := st.announceHost(); err != nil { t.Fatal(err) } // Create tmp directory for uploads/downloads. walletTestDir := build.TempDir("wallet_relative_path_backup") err = os.MkdirAll(walletTestDir, 0700) if err != nil { t.Fatal(err) } // Wallet backup should error if its destination is a relative path backupAbsoluteError := "error when calling /wallet/backup: destination must be an absolute path" // This should error. err = st.stdGetAPI("/wallet/backup?destination=test_wallet.backup") if err == nil || err.Error() != backupAbsoluteError { t.Fatal(err) } // This as well. err = st.stdGetAPI("/wallet/backup?destination=../test_wallet.backup") if err == nil || err.Error() != backupAbsoluteError { t.Fatal(err) } // This should succeed. err = st.stdGetAPI("/wallet/backup?destination=" + filepath.Join(walletTestDir, "test_wallet.backup")) if err != nil { t.Fatal(err) } // Make sure the backup was actually created. _, errStat := os.Stat(filepath.Join(walletTestDir, "test_wallet.backup")) if errStat != nil { t.Error(errStat) } } // Tests that the /wallet/033x call checks for relative paths. func TestWalletRelativePathError033x(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestWalletRelativePathError033x") if err != nil { t.Fatal(err) } defer st.server.Close() // Announce the host. if err := st.announceHost(); err != nil { t.Fatal(err) } // Create tmp directory for uploads/downloads. walletTestDir := build.TempDir("wallet_relative_path_033x") err = os.MkdirAll(walletTestDir, 0700) if err != nil { t.Fatal(err) } // Wallet loading from 033x should error if its source is a relative path load033xAbsoluteError := "error when calling /wallet/033x: source must be an absolute path" rawSeed, _, err := st.wallet.PrimarySeed() if err != nil { t.Fatal(err) } // This is not the actual wallet password. The createServerTester doesn't // return the string password. So for the sucess test we check if we make // it past the absolute value check and instead error because of the key. seed, err := modules.SeedToString(rawSeed, "english") if err != nil { t.Fatal(err) } // This should fail. load033xValues := url.Values{} load033xValues.Set("source", "test.dat") load033xValues.Set("encryptionpassword", seed) err = st.stdPostAPI("/wallet/033x", load033xValues) if err == nil || err.Error() != load033xAbsoluteError { t.Fatal(err) } // As should this. load033xValues = url.Values{} load033xValues.Set("source", "../test.dat") load033xValues.Set("encryptionpassword", seed) err = st.stdPostAPI("/wallet/033x", load033xValues) if err == nil || err.Error() != load033xAbsoluteError { t.Fatal(err) } // This should succeed. load033xValues = url.Values{} if err = createRandFile(filepath.Join(walletTestDir, "test.dat"), 0); err != nil { t.Fatal(err) } load033xValues.Set("source", filepath.Join(walletTestDir, "test.dat")) load033xValues.Set("encryptionpassword", seed) err = st.stdPostAPI("/wallet/033x", load033xValues) if err == nil || err.Error() != "provided encryption key is incorrect" { t.Fatal(err) } } // Tests that the /wallet/siagkey call checks for relative paths. func TestWalletRelativePathErrorSiag(t *testing.T) { if testing.Short() { t.SkipNow() } st, err := createServerTester("TestWalletRelativePathErrorSiag") if err != nil { t.Fatal(err) } defer st.server.Close() // Announce the host. if err := st.announceHost(); err != nil { t.Fatal(err) } // Create tmp directory for uploads/downloads. walletTestDir := build.TempDir("wallet_relative_path_sig") err = os.MkdirAll(walletTestDir, 0700) if err != nil { t.Fatal(err) } // Wallet loading from siag should error if its source is a relative path loadSiagAbsoluteError := "error when calling /wallet/siagkey: keyfiles contains a non-absolute path" rawSeed, _, err := st.wallet.PrimarySeed() if err != nil { t.Fatal(err) } // This is not the actual wallet password. The createServerTester does not // return the string password. So for the sucess tests we check if we make // it past the absolute value check and instead error because of the key. seed, err := modules.SeedToString(rawSeed, "english") if err != nil { t.Fatal(err) } // This should fail. loadSiagValues := url.Values{} loadSiagValues.Set("keyfiles", "test.dat") loadSiagValues.Set("encryptionpassword", seed) err = st.stdPostAPI("/wallet/siagkey", loadSiagValues) if err == nil || err.Error() != loadSiagAbsoluteError { t.Fatal(err) } // As should this. loadSiagValues = url.Values{} loadSiagValues.Set("keyfiles", "../test.dat") loadSiagValues.Set("encryptionpassword", seed) err = st.stdPostAPI("/wallet/siagkey", loadSiagValues) if err == nil || err.Error() != loadSiagAbsoluteError { t.Fatal(err) } // This should fail. loadSiagValues = url.Values{} loadSiagValues.Set("keyfiles", "/test.dat,test.dat,../test.dat") loadSiagValues.Set("encryptionpassword", seed) err = st.stdPostAPI("/wallet/siagkey", loadSiagValues) if err == nil || err.Error() != loadSiagAbsoluteError { t.Fatal(err) } // As should this. loadSiagValues = url.Values{} loadSiagValues.Set("keyfiles", "../test.dat,/test.dat") loadSiagValues.Set("encryptionpassword", seed) err = st.stdPostAPI("/wallet/siagkey", loadSiagValues) if err == nil || err.Error() != loadSiagAbsoluteError { t.Fatal(err) } // This should succeed. loadSiagValues = url.Values{} if err = createRandFile(filepath.Join(walletTestDir, "test.dat"), 0); err != nil { t.Fatal(err) } loadSiagValues.Set("keyfiles", filepath.Join(walletTestDir, "test.dat")) loadSiagValues.Set("encryptionpassword", seed) err = st.stdPostAPI("/wallet/siagkey", loadSiagValues) if err == nil || err.Error() != "error when calling /wallet/siagkey: provided encryption key is incorrect" { t.Fatal(err) } // As should this. loadSiagValues = url.Values{} if err = createRandFile(filepath.Join(walletTestDir, "test1.dat"), 0); err != nil { t.Fatal(err) } loadSiagValues.Set("keyfiles", filepath.Join(walletTestDir, "test.dat")+","+filepath.Join(walletTestDir, "test1.dat")) loadSiagValues.Set("encryptionpassword", seed) err = st.stdPostAPI("/wallet/siagkey", loadSiagValues) if err == nil || err.Error() != "error when calling /wallet/siagkey: provided encryption key is incorrect" { t.Fatal(err) } } Sia-1.0.4/appveyor.yml000066400000000000000000000005311302033070500146100ustar00rootroot00000000000000version: "{build}" platform: x64 clone_folder: c:\GOPATH\src\github.com\NebulousLabs\Sia shallow_clone: true environment: GOPATH: c:\GOPATH branches: only: - master install: - go version - go env - go get -d ./... build_script: - go test -tags="testing debug" -short ./... - go test -tags="testing debug" -v -race ./... Sia-1.0.4/build/000077500000000000000000000000001302033070500133205ustar00rootroot00000000000000Sia-1.0.4/build/critical.go000066400000000000000000000020301302033070500154340ustar00rootroot00000000000000package build import ( "fmt" "os" "runtime/debug" ) // Critical should be called if a sanity check has failed, indicating developer // error. Critical is called with an extended message guiding the user to the // issue tracker on Github. If the program does not panic, the call stack for // the running goroutine is printed to help determine the error. func Critical(v ...interface{}) { s := "Critical error: " + fmt.Sprintln(v...) + "Please submit a bug report here: https://github.com/NebulousLabs/Sia/issues\n" os.Stderr.WriteString(s) if DEBUG { panic(s) } debug.PrintStack() } // Severe will print a message to os.Stderr. If DEBUG has been set panic will // be called as well. Severe should be called in situations which indicate // significant problems for the user (such as disk failure or random number // generation failure), but where crashing is not strictly required to preserve // integrity. func Severe(v ...interface{}) { s := "Severe error: " + fmt.Sprintln(v...) os.Stderr.WriteString(s) if DEBUG { panic(s) } } Sia-1.0.4/build/critical_test.go000066400000000000000000000030711302033070500165010ustar00rootroot00000000000000package build import ( "testing" ) // TestCritical checks that a panic is called in debug mode. func TestCritical(t *testing.T) { k0 := "critical test killstring" killstring := "Critical error: critical test killstring\nPlease submit a bug report here: https://github.com/NebulousLabs/Sia/issues\n" defer func() { r := recover() if r != killstring { t.Error("panic did not work:", r, killstring) } }() Critical(k0) } // TestCriticalVariadic checks that a panic is called in debug mode. func TestCriticalVariadic(t *testing.T) { k0 := "variadic" k1 := "critical" k2 := "test" k3 := "killstring" killstring := "Critical error: variadic critical test killstring\nPlease submit a bug report here: https://github.com/NebulousLabs/Sia/issues\n" defer func() { r := recover() if r != killstring { t.Error("panic did not work:", r, killstring) } }() Critical(k0, k1, k2, k3) } // TestSevere checks that a panic is called in debug mode. func TestSevere(t *testing.T) { k0 := "severe test killstring" killstring := "Severe error: severe test killstring\n" defer func() { r := recover() if r != killstring { t.Error("panic did not work:", r, killstring) } }() Severe(k0) } // TestSevereVariadic checks that a panic is called in debug mode. func TestSevereVariadic(t *testing.T) { k0 := "variadic" k1 := "severe" k2 := "test" k3 := "killstring" killstring := "Severe error: variadic severe test killstring\n" defer func() { r := recover() if r != killstring { t.Error("panic did not work:", r, killstring) } }() Severe(k0, k1, k2, k3) } Sia-1.0.4/build/debug_off.go000066400000000000000000000000651302033070500155700ustar00rootroot00000000000000// +build !debug package build const DEBUG = false Sia-1.0.4/build/debug_on.go000066400000000000000000000000631302033070500154300ustar00rootroot00000000000000// +build debug package build const DEBUG = true Sia-1.0.4/build/errors.go000066400000000000000000000010371302033070500151640ustar00rootroot00000000000000package build import ( "errors" "strings" ) // JoinErrors concatenates the elements of errs to create a single error. The // separator string sep is placed between elements in the resulting error. Nil // errors are skipped. If errs is empty or only contains nil elements, // JoinErrors returns nil. func JoinErrors(errs []error, sep string) error { var strs []string for _, err := range errs { if err != nil { strs = append(strs, err.Error()) } } if len(strs) > 0 { return errors.New(strings.Join(strs, sep)) } return nil } Sia-1.0.4/build/errors_test.go000066400000000000000000000033371302033070500162300ustar00rootroot00000000000000package build import ( "errors" "testing" ) // TestJoinErrors tests that JoinErrors only returns non-nil when there are // non-nil elements in errs. And tests that the returned error's string the // concatenation of all the strings of the elements in errs, in order and // separated by sep. func TestJoinErrors(t *testing.T) { tests := []struct { errs []error sep string wantNil bool errStrWant string }{ // Test that JoinErrors returns nil when errs is nil. { wantNil: true, }, // Test that JoinErrors returns nil when errs is an empty slice. { errs: []error{}, wantNil: true, }, // Test that JoinErrors returns nil when errs has only nil elements. { errs: []error{nil}, wantNil: true, }, { errs: []error{nil, nil, nil}, wantNil: true, }, // Test that JoinErrors returns non-nil with the expected string when errs has only one non-nil element. { errs: []error{errors.New("foo")}, sep: ";", errStrWant: "foo", }, // Test that JoinErrors returns non-nil with the expected string when errs has multiple non-nil elements. { errs: []error{errors.New("foo"), errors.New("bar"), errors.New("baz")}, sep: ";", errStrWant: "foo;bar;baz", }, // Test that nil errors are ignored. { errs: []error{nil, errors.New("foo"), nil, nil, nil, errors.New("bar"), errors.New("baz"), nil, nil, nil}, sep: ";", errStrWant: "foo;bar;baz", }, } for _, tt := range tests { err := JoinErrors(tt.errs, tt.sep) if tt.wantNil && err != nil { t.Errorf("expected nil error, got '%v'", err) } else if err != nil && err.Error() != tt.errStrWant { t.Errorf("expected '%v', got '%v'", tt.errStrWant, err) } } } Sia-1.0.4/build/release_dev.go000066400000000000000000000000641302033070500161250ustar00rootroot00000000000000// +build dev package build const Release = "dev" Sia-1.0.4/build/release_standard.go000066400000000000000000000001031302033070500171410ustar00rootroot00000000000000// +build !testing,!dev package build const Release = "standard" Sia-1.0.4/build/release_testing.go000066400000000000000000000000741302033070500170250ustar00rootroot00000000000000// +build testing package build const Release = "testing" Sia-1.0.4/build/testing.go000066400000000000000000000030151302033070500153230ustar00rootroot00000000000000package build import ( "errors" "io" "io/ioutil" "os" "path/filepath" ) var ( // SiaTestingDir is the directory that contains all of the files and // folders created during testing. SiaTestingDir = filepath.Join(os.TempDir(), "SiaTesting") ) // TempDir joins the provided directories and prefixes them with the Sia // testing directory. func TempDir(dirs ...string) string { path := filepath.Join(SiaTestingDir, filepath.Join(dirs...)) os.RemoveAll(path) // remove old test data return path } // CopyFile copies a file from a source to a destination. func CopyFile(source, dest string) error { sf, err := os.Open(source) if err != nil { return err } defer sf.Close() df, err := os.Create(dest) if err != nil { return err } defer df.Close() _, err = io.Copy(df, sf) if err != nil { return err } return nil } // CopyDir copies a directory and all of its contents to the destination // directory. func CopyDir(source, dest string) error { stat, err := os.Stat(source) if err != nil { return err } if !stat.IsDir() { return errors.New("source is not a directory") } err = os.MkdirAll(dest, stat.Mode()) if err != nil { return err } files, err := ioutil.ReadDir(source) for _, file := range files { newSource := filepath.Join(source, file.Name()) newDest := filepath.Join(dest, file.Name()) if file.IsDir() { err = CopyDir(newSource, newDest) if err != nil { return err } } else { err = CopyFile(newSource, newDest) if err != nil { return err } } } return nil } Sia-1.0.4/build/testing_test.go000066400000000000000000000025331302033070500163660ustar00rootroot00000000000000package build import ( "bytes" "crypto/rand" "io/ioutil" "os" "path/filepath" "testing" ) // TestCopyDir checks that CopyDir copies directories as expected. func TestCopyDir(t *testing.T) { // Create some nested folders to copy. os.MkdirAll(TempDir("build"), 0700) root := TempDir("build", "TestCopyDir") os.MkdirAll(root, 0700) data := make([][]byte, 2) for i := range data { data[i] = make([]byte, 4e3) _, err := rand.Read(data[i]) if err != nil { t.Fatal(err) } } // Create a file and a directory. err := ioutil.WriteFile(filepath.Join(root, "f1"), data[0], 0700) if err != nil { t.Fatal(err) } err = os.MkdirAll(filepath.Join(root, "d1"), 0700) if err != nil { t.Fatal(err) } err = ioutil.WriteFile(filepath.Join(root, "d1", "d1f1"), data[1], 0700) if err != nil { t.Fatal(err) } // Copy the root directory. rootCopy := root + "-copied" err = CopyDir(root, rootCopy) if err != nil { t.Fatal(err) } // Verify that the two files, and dir with two files are all correctly // copied. f1, err := ioutil.ReadFile(filepath.Join(rootCopy, "f1")) if err != nil { t.Fatal(err) } if !bytes.Equal(f1, data[0]) { t.Error("f1 did not match") } d1f1, err := ioutil.ReadFile(filepath.Join(rootCopy, "d1", "d1f1")) if err != nil { t.Fatal(err) } if !bytes.Equal(d1f1, data[1]) { t.Error("f1 did not match") } } Sia-1.0.4/build/version.go000066400000000000000000000031721302033070500153370ustar00rootroot00000000000000package build import ( "strconv" "strings" ) const ( // Version is the current version of siad. Version = "1.0.4" // MaxEncodedVersionLength is the maximum length of a version string encoded // with the encode package. 100 is much larger than any version number we send // now, but it allows us to send additional information in the version string // later if we choose. For example appending the version string with the HEAD // commit hash. MaxEncodedVersionLength = 100 ) // IsVersion returns whether str is a valid version number. func IsVersion(str string) bool { for _, n := range strings.Split(str, ".") { if _, err := strconv.Atoi(n); err != nil { return false } } return true } // min returns the smaller of two integers. func min(a, b int) int { if a < b { return a } return b } // VersionCmp returns an int indicating the difference between a and b. It // follows the convention of bytes.Compare and big.Cmp: // // -1 if a < b // 0 if a == b // +1 if a > b // // One important quirk is that "1.1.0" is considered newer than "1.1", despite // being numerically equal. func VersionCmp(a, b string) int { aNums := strings.Split(a, ".") bNums := strings.Split(b, ".") for i := 0; i < min(len(aNums), len(bNums)); i++ { // assume that both version strings are valid aInt, _ := strconv.Atoi(aNums[i]) bInt, _ := strconv.Atoi(bNums[i]) if aInt < bInt { return -1 } else if aInt > bInt { return 1 } } // all shared digits are equal, but lengths may not be equal if len(aNums) < len(bNums) { return -1 } else if len(aNums) > len(bNums) { return 1 } // strings are identical return 0 } Sia-1.0.4/build/version_test.go000066400000000000000000000020151302033070500163710ustar00rootroot00000000000000package build import ( "testing" ) // TestVersionCmp checks that in all cases, VersionCmp returns the correct // result. func TestVersionCmp(t *testing.T) { versionTests := []struct { a, b string exp int }{ {"0.1", "0.0.9", 1}, {"0.1", "0.1", 0}, {"0.1", "0.1.1", -1}, {"0.1", "0.1.0", -1}, {"0.1", "1.1", -1}, {"0.1.1.0", "0.1.1", 1}, } for _, test := range versionTests { if actual := VersionCmp(test.a, test.b); actual != test.exp { t.Errorf("Comparing %v to %v should return %v (got %v)", test.a, test.b, test.exp, actual) } } } // TestIsVersion tests the IsVersion function. func TestIsVersion(t *testing.T) { versionTests := []struct { str string exp bool }{ {"1.0", true}, {"1", true}, {"0.1.2.3.4.5", true}, {"foo", false}, {".1", false}, {"1.", false}, {"a.b", false}, {"1.o", false}, {".", false}, {"", false}, } for _, test := range versionTests { if IsVersion(test.str) != test.exp { t.Errorf("IsVersion(%v) should return %v", test.str, test.exp) } } } Sia-1.0.4/compatibility/000077500000000000000000000000001302033070500150725ustar00rootroot00000000000000Sia-1.0.4/compatibility/compatibility.go000066400000000000000000000002271302033070500202730ustar00rootroot00000000000000// package compatibility contains a bunch of tests to make sure that // compatibility has been preserved with previous versions. package compatibility Sia-1.0.4/compatibility/siafile_v0.4.8.sia000066400000000000000000000002241302033070500201170ustar00rootroot00000000000000Sia Shared File0.4 na⒴̜T]C c'f&XߧMm\_ٷ#/ǘcfbt#nugwNvpd9+af@XQ2# ˆSia-1.0.4/compatibility/siag_1.0_1of1_Key0.siakey000066400000000000000000000002571302033070500213240ustar00rootroot00000000000000siag1.0 $>1Q_ثˇ ȳ~f½o%aWvz` r_~s-VWSHHӅ;h}MNޮ/6Y Ged25519 Ӆ;h}MNޮ/6Y Ged25519 R<]?a8+0`d=߄(gπSdSia-1.0.4/compatibility/siag_1.0_1of2_Key1.siakey000066400000000000000000000003471302033070500213260ustar00rootroot00000000000000siag1.0vutG;X-'iGp$:R<]?a8+0`d=߄(gπSded25519 Ӆ;h}MNޮ/6Y Ged25519 R<]?a8+0`d=߄(gπSdSia-1.0.4/compatibility/siag_1.0_2of3_Key0.siakey000066400000000000000000000004371302033070500213270ustar00rootroot00000000000000siag1.0P?7Cֆ+]nГjOWc!j i le o_tqF jued25519  le o_tqF jued25519 6e%/ȴB=OgVOz%6ed25519 Z 6q)ϴ'N2(G2Sia-1.0.4/compatibility/siag_1.0_2of3_Key1.siakey000066400000000000000000000004371302033070500213300ustar00rootroot00000000000000siag1.0e!G e ]yhbK( E6e%/ȴB=OgVOz%6ed25519  le o_tqF jued25519 6e%/ȴB=OgVOz%6ed25519 Z 6q)ϴ'N2(G2Sia-1.0.4/compatibility/siag_1.0_2of3_Key2.siakey000066400000000000000000000004371302033070500213310ustar00rootroot00000000000000siag1.0 1QpYMPO PIqperZPZ 6q)ϴ'N2(G2ed25519  le o_tqF jued25519 6e%/ȴB=OgVOz%6ed25519 Z 6q)ϴ'N2(G2Sia-1.0.4/compatibility/siag_1.0_3of3_Key0.siakey000066400000000000000000000004371302033070500213300ustar00rootroot00000000000000siag1.0ց{ʖaݤR2ijV^iY>a/=x8Tp㤮/ed25519 iY>a/=x8Tp㤮/ed25519 h 13NlDaεQQ ed25519 x{7,!J4u=> =Sia-1.0.4/compatibility/siag_1.0_3of3_Key1.siakey000066400000000000000000000004371302033070500213310ustar00rootroot00000000000000siag1.0v*oˤiopiܲlRw I1eh 13NlDaεQQ ed25519 iY>a/=x8Tp㤮/ed25519 h 13NlDaεQQ ed25519 x{7,!J4u=> =Sia-1.0.4/compatibility/siag_1.0_3of3_Key2.siakey000066400000000000000000000004371302033070500213320ustar00rootroot00000000000000siag1.0´{Dܻ"(ԋqჃ画S6ƪx{7,!J4u=> =ed25519 iY>a/=x8Tp㤮/ed25519 h 13NlDaεQQ ed25519 x{7,!J4u=> =Sia-1.0.4/compatibility/siag_1.0_4of9_Key0.siakey000066400000000000000000000011571302033070500213370ustar00rootroot00000000000000siag1.0Xd1§fu#Br fڬa 8 8ZPEBVq, ed25519 r fڬa 8 8ZPEBVq,ed25519 HHphE8Dg)DFOUed25519 z0$k5$y$j TW9wkG_ed25519 ϼ_o+Nx>"Red25519 SWxWp{ϼE2qXzj*ed25519 d7Exk_. f`3 a^fed25519 :c"P{c׏Z1d[5Hnڱt-ed25519 *~wL8a"U?-&}Ted25519 s/y_y5ǘ֍Cr(,lSia-1.0.4/compatibility/siag_1.0_4of9_Key1.siakey000066400000000000000000000011571302033070500213400ustar00rootroot00000000000000siag1.0:Z-ąLDll`r>>EdHHphE8Dg)DFOU ed25519 r fڬa 8 8ZPEBVq,ed25519 HHphE8Dg)DFOUed25519 z0$k5$y$j TW9wkG_ed25519 ϼ_o+Nx>"Red25519 SWxWp{ϼE2qXzj*ed25519 d7Exk_. f`3 a^fed25519 :c"P{c׏Z1d[5Hnڱt-ed25519 *~wL8a"U?-&}Ted25519 s/y_y5ǘ֍Cr(,lSia-1.0.4/compatibility/siag_1.0_4of9_Key2.siakey000066400000000000000000000011571302033070500213410ustar00rootroot00000000000000siag1.0 1;`<P K^8mSUZvz0$k5$y$j TW9wkG_ ed25519 r fڬa 8 8ZPEBVq,ed25519 HHphE8Dg)DFOUed25519 z0$k5$y$j TW9wkG_ed25519 ϼ_o+Nx>"Red25519 SWxWp{ϼE2qXzj*ed25519 d7Exk_. f`3 a^fed25519 :c"P{c׏Z1d[5Hnڱt-ed25519 *~wL8a"U?-&}Ted25519 s/y_y5ǘ֍Cr(,lSia-1.0.4/compatibility/siag_1.0_4of9_Key3.siakey000066400000000000000000000011571302033070500213420ustar00rootroot00000000000000siag1.0VV%RF8JE-Ν^9ϼ_o+Nx>"R ed25519 r fڬa 8 8ZPEBVq,ed25519 HHphE8Dg)DFOUed25519 z0$k5$y$j TW9wkG_ed25519 ϼ_o+Nx>"Red25519 SWxWp{ϼE2qXzj*ed25519 d7Exk_. f`3 a^fed25519 :c"P{c׏Z1d[5Hnڱt-ed25519 *~wL8a"U?-&}Ted25519 s/y_y5ǘ֍Cr(,lSia-1.0.4/compatibility/siag_1.0_4of9_Key4.siakey000066400000000000000000000011571302033070500213430ustar00rootroot00000000000000siag1.0(Z *l9"I8#SWxWp{ϼE2qXzj* ed25519 r fڬa 8 8ZPEBVq,ed25519 HHphE8Dg)DFOUed25519 z0$k5$y$j TW9wkG_ed25519 ϼ_o+Nx>"Red25519 SWxWp{ϼE2qXzj*ed25519 d7Exk_. f`3 a^fed25519 :c"P{c׏Z1d[5Hnڱt-ed25519 *~wL8a"U?-&}Ted25519 s/y_y5ǘ֍Cr(,lSia-1.0.4/compatibility/siag_1.0_4of9_Key5.siakey000066400000000000000000000011571302033070500213440ustar00rootroot00000000000000siag1.0,a^qRǝHK,IDd7Exk_. f`3 a^f ed25519 r fڬa 8 8ZPEBVq,ed25519 HHphE8Dg)DFOUed25519 z0$k5$y$j TW9wkG_ed25519 ϼ_o+Nx>"Red25519 SWxWp{ϼE2qXzj*ed25519 d7Exk_. f`3 a^fed25519 :c"P{c׏Z1d[5Hnڱt-ed25519 *~wL8a"U?-&}Ted25519 s/y_y5ǘ֍Cr(,lSia-1.0.4/compatibility/siag_1.0_4of9_Key6.siakey000066400000000000000000000011571302033070500213450ustar00rootroot00000000000000siag1.0ǸT\"6c1<9臇M~:c"P{c׏Z1d[5Hnڱt- ed25519 r fڬa 8 8ZPEBVq,ed25519 HHphE8Dg)DFOUed25519 z0$k5$y$j TW9wkG_ed25519 ϼ_o+Nx>"Red25519 SWxWp{ϼE2qXzj*ed25519 d7Exk_. f`3 a^fed25519 :c"P{c׏Z1d[5Hnڱt-ed25519 *~wL8a"U?-&}Ted25519 s/y_y5ǘ֍Cr(,lSia-1.0.4/compatibility/siag_1.0_4of9_Key7.siakey000066400000000000000000000011571302033070500213460ustar00rootroot00000000000000siag1.0>c9ƶhIy(r;{Yy*~wL8a"U?-&}T ed25519 r fڬa 8 8ZPEBVq,ed25519 HHphE8Dg)DFOUed25519 z0$k5$y$j TW9wkG_ed25519 ϼ_o+Nx>"Red25519 SWxWp{ϼE2qXzj*ed25519 d7Exk_. f`3 a^fed25519 :c"P{c׏Z1d[5Hnڱt-ed25519 *~wL8a"U?-&}Ted25519 s/y_y5ǘ֍Cr(,lSia-1.0.4/compatibility/siag_1.0_4of9_Key8.siakey000066400000000000000000000011571302033070500213470ustar00rootroot00000000000000siag1.0} aڄܢؐ"$mLjr"s/y_y5ǘ֍Cr(,l ed25519 r fڬa 8 8ZPEBVq,ed25519 HHphE8Dg)DFOUed25519 z0$k5$y$j TW9wkG_ed25519 ϼ_o+Nx>"Red25519 SWxWp{ϼE2qXzj*ed25519 d7Exk_. f`3 a^fed25519 :c"P{c׏Z1d[5Hnڱt-ed25519 *~wL8a"U?-&}Ted25519 s/y_y5ǘ֍Cr(,lSia-1.0.4/compatibility/siag_1.0_test.go000066400000000000000000000062661302033070500177730ustar00rootroot00000000000000package compatibility // siag.go checks that any changes made to the code retain compatibility with // old versions of siag. import ( "errors" "path/filepath" "strconv" "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/types" ) // KeyPairSiag_1_0 matches the KeyPair struct of the siag 1.0 code. type KeyPairSiag_1_0 struct { Header string Version string Index int SecretKey crypto.SecretKey UnlockConditions types.UnlockConditions } // verifyKeysSiag_1_0 is a copy-pasted version of the verifyKeys method // from siag 1.0. func verifyKeysSiag_1_0(uc types.UnlockConditions, folder string, keyname string) error { keysRequired := uc.SignaturesRequired totalKeys := uint64(len(uc.PublicKeys)) loadedKeys := make([]KeyPairSiag_1_0, totalKeys) for i := 0; i < len(loadedKeys); i++ { err := encoding.ReadFile(filepath.Join(folder, keyname+"_Key"+strconv.Itoa(i)+".siakey"), &loadedKeys[i]) if err != nil { return err } } for _, loadedKey := range loadedKeys { if loadedKey.UnlockConditions.UnlockHash() != uc.UnlockHash() { return errors.New("ErrCorruptedKey") } } txn := types.Transaction{ SiafundInputs: []types.SiafundInput{ { UnlockConditions: loadedKeys[0].UnlockConditions, }, }, } var i uint64 for i != totalKeys { if i+keysRequired > totalKeys { i = totalKeys - keysRequired } var j uint64 for j < keysRequired { txn.TransactionSignatures = append(txn.TransactionSignatures, types.TransactionSignature{ PublicKeyIndex: i, CoveredFields: types.CoveredFields{WholeTransaction: true}, }) sigHash := txn.SigHash(int(j)) sig, err := crypto.SignHash(sigHash, loadedKeys[i].SecretKey) if err != nil { return err } txn.TransactionSignatures[j].Signature = sig[:] i++ j++ } err := txn.StandaloneValid(0) if err != nil { return err } txn.TransactionSignatures = nil } return nil } // TestVerifyKeysSiag_1_0 loads some keys generated by siag1.0. // Verification must still work. func TestVerifyKeysSiag_1_0(t *testing.T) { if testing.Short() { t.SkipNow() } var kp KeyPairSiag_1_0 // 1 of 1 err := encoding.ReadFile("siag_1.0_1of1_Key0.siakey", &kp) if err != nil { t.Fatal(err) } err = verifyKeysSiag_1_0(kp.UnlockConditions, "", "siag_1.0_1of1") if err != nil { t.Fatal(err) } // 1 of 2 err = encoding.ReadFile("siag_1.0_1of2_Key0.siakey", &kp) if err != nil { t.Fatal(err) } err = verifyKeysSiag_1_0(kp.UnlockConditions, "", "siag_1.0_1of2") if err != nil { t.Fatal(err) } // 2 of 3 err = encoding.ReadFile("siag_1.0_2of3_Key0.siakey", &kp) if err != nil { t.Fatal(err) } err = verifyKeysSiag_1_0(kp.UnlockConditions, "", "siag_1.0_2of3") if err != nil { t.Fatal(err) } // 3 of 3 err = encoding.ReadFile("siag_1.0_3of3_Key0.siakey", &kp) if err != nil { t.Fatal(err) } err = verifyKeysSiag_1_0(kp.UnlockConditions, "", "siag_1.0_3of3") if err != nil { t.Fatal(err) } // 4 of 9 err = encoding.ReadFile("siag_1.0_4of9_Key0.siakey", &kp) if err != nil { t.Fatal(err) } err = verifyKeysSiag_1_0(kp.UnlockConditions, "", "siag_1.0_4of9") if err != nil { t.Fatal(err) } } Sia-1.0.4/crypto/000077500000000000000000000000001302033070500135415ustar00rootroot00000000000000Sia-1.0.4/crypto/discard.go000066400000000000000000000005311302033070500155000ustar00rootroot00000000000000package crypto // SecureWipe destroys the data contained within a byte slice. There are no // strong guarantees that all copies of the memory have been eliminated. If the // OS was doing context switching or using swap space the keys may still be // elsewhere in memory. func SecureWipe(data []byte) { for i := range data { data[i] = 0 } } Sia-1.0.4/crypto/discard_test.go000066400000000000000000000007671302033070500165520ustar00rootroot00000000000000package crypto import ( "bytes" "testing" ) // TestUnitSecureWipe tests that the SecureWipe function sets all the elements // in a byte slice to 0. func TestUnitSecureWipe(t *testing.T) { s := []byte{1, 2, 3, 4} SecureWipe(s) if !bytes.Equal(s, make([]byte, len(s))) { t.Error("some bytes not set to 0") } } // TestUnitSecureWipeEdgeCases tests that SecureWipe doesn't panic on nil or // empty slices. func TestUnitSecureWipeEdgeCases(t *testing.T) { SecureWipe(nil) SecureWipe([]byte{}) } Sia-1.0.4/crypto/encrypt.go000066400000000000000000000056241302033070500155630ustar00rootroot00000000000000package crypto // encrypt.go contains functions for encrypting and decrypting data byte slices // and readers. import ( "crypto/cipher" "crypto/rand" "encoding/json" "errors" "io" "golang.org/x/crypto/twofish" ) const ( TwofishOverhead = 28 // number of bytes added by EncryptBytes ) var ( ErrInsufficientLen = errors.New("supplied ciphertext is not long enough to contain a nonce") ) type ( Ciphertext []byte TwofishKey [EntropySize]byte ) // GenerateEncryptionKey produces a key that can be used for encrypting and // decrypting files. func GenerateTwofishKey() (key TwofishKey, err error) { _, err = rand.Read(key[:]) return key, err } // NewCipher creates a new Twofish cipher from the key. func (key TwofishKey) NewCipher() cipher.Block { // NOTE: NewCipher only returns an error if len(key) != 16, 24, or 32. cipher, _ := twofish.NewCipher(key[:]) return cipher } // EncryptBytes encrypts a []byte using the key. EncryptBytes uses GCM and // prepends the nonce (12 bytes) to the ciphertext. func (key TwofishKey) EncryptBytes(plaintext []byte) (Ciphertext, error) { // Create the cipher. // NOTE: NewGCM only returns an error if twofishCipher.BlockSize != 16. aead, _ := cipher.NewGCM(key.NewCipher()) // Create the nonce. nonce, err := RandBytes(aead.NonceSize()) if err != nil { return nil, err } // Encrypt the data. No authenticated data is provided, as EncryptBytes is // meant for file encryption. return aead.Seal(nonce, nonce, plaintext, nil), nil } // DecryptBytes decrypts the ciphertext created by EncryptBytes. The nonce is // expected to be the first 12 bytes of the ciphertext. func (key TwofishKey) DecryptBytes(ct Ciphertext) ([]byte, error) { // Create the cipher. // NOTE: NewGCM only returns an error if twofishCipher.BlockSize != 16. aead, _ := cipher.NewGCM(key.NewCipher()) // Check for a nonce. if len(ct) < aead.NonceSize() { return nil, ErrInsufficientLen } // Decrypt the data. return aead.Open(nil, ct[:aead.NonceSize()], ct[aead.NonceSize():], nil) } // NewWriter returns a writer that encrypts or decrypts its input stream. func (key TwofishKey) NewWriter(w io.Writer) io.Writer { // OK to use a zero IV if the key is unique for each ciphertext. iv := make([]byte, twofish.BlockSize) stream := cipher.NewOFB(key.NewCipher(), iv) return &cipher.StreamWriter{S: stream, W: w} } // NewReader returns a reader that encrypts or decrypts its input stream. func (key TwofishKey) NewReader(r io.Reader) io.Reader { // OK to use a zero IV if the key is unique for each ciphertext. iv := make([]byte, twofish.BlockSize) stream := cipher.NewOFB(key.NewCipher(), iv) return &cipher.StreamReader{S: stream, R: r} } func (c Ciphertext) MarshalJSON() ([]byte, error) { return json.Marshal([]byte(c)) } func (c *Ciphertext) UnmarshalJSON(b []byte) error { var umarB []byte err := json.Unmarshal(b, &umarB) if err != nil { return err } *c = Ciphertext(umarB) return nil } Sia-1.0.4/crypto/encrypt_test.go000066400000000000000000000156241302033070500166230ustar00rootroot00000000000000package crypto import ( "bytes" "compress/gzip" "crypto/rand" "testing" ) // TestTwofishEncryption checks that encryption and decryption works correctly. func TestTwofishEncryption(t *testing.T) { // Get a key for encryption. key, err := GenerateTwofishKey() if err != nil { t.Fatal(err) } // Encrypt and decrypt a zero plaintext, and compare the decrypted to the // original. plaintext := make([]byte, 600) ciphertext, err := key.EncryptBytes(plaintext) if err != nil { t.Fatal(err) } decryptedPlaintext, err := key.DecryptBytes(ciphertext) if err != nil { t.Fatal(err) } if !bytes.Equal(plaintext, decryptedPlaintext) { t.Fatal("Encrypted and decrypted zero plaintext do not match") } // Try again with a nonzero plaintext. plaintext = make([]byte, 600) _, err = rand.Read(plaintext) if err != nil { t.Fatal(err) } ciphertext, err = key.EncryptBytes(plaintext) if err != nil { t.Fatal(err) } decryptedPlaintext, err = key.DecryptBytes(ciphertext) if err != nil { t.Fatal(err) } if !bytes.Equal(plaintext, decryptedPlaintext) { t.Fatal("Encrypted and decrypted zero plaintext do not match") } // Try to decrypt using a different key key2, err := GenerateTwofishKey() if err != nil { t.Fatal(err) } _, err = key2.DecryptBytes(ciphertext) if err == nil { t.Fatal("Expecting failed authentication err", err) } // Try to decrypt using bad ciphertexts. ciphertext[0]++ _, err = key.DecryptBytes(ciphertext) if err == nil { t.Fatal("Expecting failed authentication err", err) } _, err = key.DecryptBytes(ciphertext[:10]) if err != ErrInsufficientLen { t.Error("Expecting ErrInsufficientLen:", err) } // Try to trigger a panic or error with nil values. _, err = key.EncryptBytes(nil) if err != nil { t.Error(err) } _, err = key.DecryptBytes(nil) if err != ErrInsufficientLen { t.Error("Expecting ErrInsufficientLen:", err) } } // TestReaderWriter probes the NewReader and NewWriter methods of the key type. func TestReaderWriter(t *testing.T) { // Get a key for encryption. key, err := GenerateTwofishKey() if err != nil { t.Fatal(err) } // Generate plaintext. const plaintextSize = 600 plaintext := make([]byte, plaintextSize) _, err = rand.Read(plaintext) if err != nil { t.Fatal(err) } // Create writer and encrypt plaintext. buf := new(bytes.Buffer) key.NewWriter(buf).Write(plaintext) // There should be no overhead present. if buf.Len() != plaintextSize { t.Fatalf("encryption introduced %v bytes of overhead", buf.Len()-plaintextSize) } // Create reader and decrypt ciphertext. var decrypted = make([]byte, plaintextSize) key.NewReader(buf).Read(decrypted) if !bytes.Equal(plaintext, decrypted) { t.Error("couldn't decrypt encrypted stream") } } // TestTwofishEntropy encrypts and then decrypts a zero plaintext, checking // that the ciphertext is high entropy. func TestTwofishEntropy(t *testing.T) { if testing.Short() { t.SkipNow() } // Encrypt a larger zero plaintext and make sure that the outcome is high // entropy. Entropy is measured by compressing the ciphertext with gzip. // 10 * 1000 bytes was chosen to minimize the impact of gzip overhead. const cipherSize = 10e3 key, err := GenerateTwofishKey() if err != nil { t.Fatal(err) } plaintext := make([]byte, cipherSize) ciphertext, err := key.EncryptBytes(plaintext) if err != nil { t.Fatal(err) } // Gzip the ciphertext var b bytes.Buffer zip := gzip.NewWriter(&b) _, err = zip.Write(ciphertext) if err != nil { t.Fatal(err) } zip.Close() if b.Len() < cipherSize { t.Error("supposedly high entropy ciphertext has been compressed!") } } // TestUnitCiphertextUnmarshalInvalidJSON tests that Ciphertext.UnmarshalJSON // correctly fails on invalid JSON marshalled Ciphertext. func TestUnitCiphertextUnmarshalInvalidJSON(t *testing.T) { // Test unmarshalling invalid JSON. invalidJSONBytes := [][]byte{ nil, {}, []byte("\""), } for _, jsonBytes := range invalidJSONBytes { var ct Ciphertext err := ct.UnmarshalJSON(jsonBytes) if err == nil { t.Errorf("expected unmarshall to fail on the invalid JSON: %q\n", jsonBytes) } } } // TestCiphertextMarshalling tests that marshalling Ciphertexts to JSON results // in the expected JSON. Also tests that marshalling that JSON back to // Ciphertext results in the original Ciphertext. func TestCiphertextMarshalling(t *testing.T) { // Ciphertexts and corresponding JSONs to test marshalling and // unmarshalling. ciphertextMarshallingTests := []struct { ct Ciphertext jsonBytes []byte }{ {ct: Ciphertext(nil), jsonBytes: []byte("null")}, {ct: Ciphertext(""), jsonBytes: []byte(`""`)}, {ct: Ciphertext("a ciphertext"), jsonBytes: []byte(`"YSBjaXBoZXJ0ZXh0"`) /* base64 encoding of the Ciphertext */}, } for _, test := range ciphertextMarshallingTests { expectedCt := test.ct expectedJSONBytes := test.jsonBytes // Create a copy of expectedCt so Unmarshalling does not modify it, as // we need it later for comparison. var ct Ciphertext if expectedCt == nil { ct = nil } else { ct = make(Ciphertext, len(expectedCt)) copy(ct, expectedCt) } // Marshal Ciphertext to JSON. jsonBytes, err := ct.MarshalJSON() if err != nil { t.Fatal(err) } if !bytes.Equal(jsonBytes, expectedJSONBytes) { // Use %#v instead of %v because %v prints Ciphertexts constructed // with nil and []byte{} identically. t.Fatalf("Ciphertext %#v marshalled incorrectly: expected %q, got %q\n", ct, expectedJSONBytes, jsonBytes) } // Unmarshal back to Ciphertext. err = ct.UnmarshalJSON(jsonBytes) if err != nil { t.Fatal(err) } // Compare resulting Ciphertext with expected Ciphertext. if expectedCt == nil && ct != nil || expectedCt != nil && ct == nil || !bytes.Equal(expectedCt, ct) { // Use %#v instead of %v because %v prints Ciphertexts constructed // with nil and []byte{} identically. t.Errorf("Ciphertext %#v unmarshalled incorrectly: got %#v\n", expectedCt, ct) } } } // TestTwofishNewCipherAssumption tests that the length of a TwofishKey is 16, // 24, or 32 as these are the only cases where twofish.NewCipher(key[:]) // doesn't return an error. func TestTwofishNewCipherAssumption(t *testing.T) { // Generate key. key, err := GenerateTwofishKey() if err != nil { t.Fatal(err) } // Test key length. keyLen := len(key) if keyLen != 16 && keyLen != 24 && keyLen != 32 { t.Errorf("TwofishKey must have length 16, 24, or 32, but generated key has length %d\n", keyLen) } } // TestCipherNewGCMAssumption tests that the BlockSize of a cipher block is 16, // as this is the only case where cipher.NewGCM(block) doesn't return an error. func TestCipherNewGCMAssumption(t *testing.T) { // Generate a key and then cipher block from key. key, err := GenerateTwofishKey() if err != nil { t.Fatal(err) } // Test block size. block := key.NewCipher() if block.BlockSize() != 16 { t.Errorf("cipher must have BlockSize 16, but generated cipher has BlockSize %d\n", block.BlockSize()) } } Sia-1.0.4/crypto/hash.go000066400000000000000000000054271302033070500150230ustar00rootroot00000000000000package crypto // hash.go supplies a few geneeral hashing functions, using the hashing // algorithm blake2b. Because changing the hashing algorithm for Sia has much // stronger implications than changing any of the other algorithms, blake2b is // the only supported algorithm. Sia is not really flexible enough to support // multiple. import ( "bytes" "encoding/hex" "encoding/json" "errors" "hash" "github.com/NebulousLabs/Sia/encoding" "github.com/dchest/blake2b" ) const ( HashSize = 32 ) type ( Hash [HashSize]byte // HashSlice is used for sorting HashSlice []Hash ) var ( ErrHashWrongLen = errors.New("encoded value has the wrong length to be a hash") ) // NewHash returns a blake2b 256bit hasher. func NewHash() hash.Hash { return blake2b.New256() } // HashAll takes a set of objects as input, encodes them all using the encoding // package, and then hashes the result. func HashAll(objs ...interface{}) Hash { return HashBytes(encoding.MarshalAll(objs...)) } // HashBytes takes a byte slice and returns the result. func HashBytes(data []byte) Hash { return Hash(blake2b.Sum256(data)) } // HashObject takes an object as input, encodes it using the encoding package, // and then hashes the result. func HashObject(obj interface{}) Hash { return HashBytes(encoding.Marshal(obj)) } // These functions implement sort.Interface, allowing hashes to be sorted. func (hs HashSlice) Len() int { return len(hs) } func (hs HashSlice) Less(i, j int) bool { return bytes.Compare(hs[i][:], hs[j][:]) < 0 } func (hs HashSlice) Swap(i, j int) { hs[i], hs[j] = hs[j], hs[i] } // LoadString takes a string, parses the hash value of the string, and sets the // value of the hash equal to the hash value of the string. func (h *Hash) LoadString(s string) error { // *2 because there are 2 hex characters per byte. if len(s) != HashSize*2 { return ErrHashWrongLen } hBytes, err := hex.DecodeString(s) if err != nil { return errors.New("could not unmarshal hash: " + err.Error()) } copy(h[:], hBytes) return nil } // MarshalJSON marshales a hash as a hex string. func (h Hash) MarshalJSON() ([]byte, error) { return json.Marshal(h.String()) } // String prints the hash in hex. func (h Hash) String() string { return hex.EncodeToString(h[:]) } // UnmarshalJSON decodes the json hex string of the hash. func (h *Hash) UnmarshalJSON(b []byte) error { // *2 because there are 2 hex characters per byte. // +2 because the encoded JSON string has a `"` added at the beginning and end. if len(b) != HashSize*2+2 { return ErrHashWrongLen } // b[1 : len(b)-1] cuts off the leading and trailing `"` in the JSON string. hBytes, err := hex.DecodeString(string(b[1 : len(b)-1])) if err != nil { return errors.New("could not unmarshal hash: " + err.Error()) } copy(h[:], hBytes) return nil } Sia-1.0.4/crypto/hash_test.go000066400000000000000000000103341302033070500160530ustar00rootroot00000000000000package crypto import ( "bytes" "crypto/rand" "encoding/json" "sort" "strings" "testing" ) type ( // TestObject is a struct that's used for testing HashAll and HashObject. The // fields have to be exported so the encoder can read them. TestObject struct { A int B byte C bool D string } ) // TestHashing uses each of the functions in hash.go and verifies that the // results are as expected. func TestHashing(t *testing.T) { // Create a test object. to := TestObject{ A: 12345, B: 5, C: true, D: "testing", } // Call HashObject on the object. var emptyHash Hash h0 := HashObject(to) if h0 == emptyHash { t.Error("HashObject returned the zero hash!") } // Call HashAll on the test object and some other fields. h1 := HashAll( int(122), byte(115), string("test"), to, ) if h1 == emptyHash { t.Error("HashObject returned the zero hash!") } // Call HashBytes on a random byte slice. data := make([]byte, 435) rand.Read(data) h2 := HashBytes(data) if h2 == emptyHash { t.Error("HashObject returned the zero hash!") } } // TestHashSorting takes a set of hashses and checks that they can be sorted. func TestHashSorting(t *testing.T) { // Created an unsorted list of hashes. hashes := make([]Hash, 5) hashes[0][0] = 12 hashes[1][0] = 7 hashes[2][0] = 13 hashes[3][0] = 14 hashes[4][0] = 1 // Sort the hashes. sort.Sort(HashSlice(hashes)) if hashes[0][0] != 1 { t.Error("bad sort") } if hashes[1][0] != 7 { t.Error("bad sort") } if hashes[2][0] != 12 { t.Error("bad sort") } if hashes[3][0] != 13 { t.Error("bad sort") } if hashes[4][0] != 14 { t.Error("bad sort") } } // TestUnitHashMarshalJSON tests that Hashes are correctly marshalled to JSON. func TestUnitHashMarshalJSON(t *testing.T) { h := HashObject("an object") jsonBytes, err := h.MarshalJSON() if err != nil { t.Fatal(err) } if !bytes.Equal(jsonBytes, []byte(`"`+h.String()+`"`)) { t.Errorf("hash %s encoded incorrectly: got %s\n", h, jsonBytes) } } // TestUnitHashUnmarshalJSON tests that unmarshalling invalid JSON will result // in an error. func TestUnitHashUnmarshalJSON(t *testing.T) { // Test unmarshalling invalid data. invalidJSONBytes := [][]byte{ // Invalid JSON. nil, {}, []byte("\""), // JSON of wrong length. []byte(""), []byte(`"` + strings.Repeat("a", HashSize*2-1) + `"`), []byte(`"` + strings.Repeat("a", HashSize*2+1) + `"`), // JSON of right length but invalid Hashes. []byte(`"` + strings.Repeat("z", HashSize*2) + `"`), []byte(`"` + strings.Repeat(".", HashSize*2) + `"`), []byte(`"` + strings.Repeat("\n", HashSize*2) + `"`), } for _, jsonBytes := range invalidJSONBytes { var h Hash err := h.UnmarshalJSON(jsonBytes) if err == nil { t.Errorf("expected unmarshall to fail on the invalid JSON: %q\n", jsonBytes) } } // Test unmarshalling valid data. expectedH := HashObject("an object") jsonBytes := []byte(`"` + expectedH.String() + `"`) var h Hash err := h.UnmarshalJSON(jsonBytes) if err != nil { t.Fatal(err) } if !bytes.Equal(h[:], expectedH[:]) { t.Errorf("Hash %s marshalled incorrectly: got %s\n", expectedH, h) } } // TestHashMarshalling checks that the marshalling of the hash type works as // expected. func TestHashMarshalling(t *testing.T) { h := HashObject("an object") hBytes, err := json.Marshal(h) if err != nil { t.Fatal(err) } var uMarH Hash err = uMarH.UnmarshalJSON(hBytes) if err != nil { t.Fatal(err) } if h != uMarH { t.Error("encoded and decoded hash do not match!") } } // TestHashLoadString checks that the LoadString method of the hash function is // working properly. func TestHashLoadString(t *testing.T) { h1 := Hash{} h2 := HashObject("tame") h1e := h1.String() h2e := h2.String() var h1d, h2d Hash err := h1d.LoadString(h1e) if err != nil { t.Fatal(err) } err = h2d.LoadString(h2e) if err != nil { t.Fatal(err) } if h1d != h1 { t.Error("decoding h1 failed") } if h2d != h2 { t.Error("decoding h2 failed") } // Try some bogus strings. h1e = h1e + "a" err = h1.LoadString(h1e) if err == nil { t.Fatal("expecting error when decoding hash of too large length") } h1e = h1e[:60] err = h1.LoadString(h1e) if err == nil { t.Fatal("expecting error when decoding hash of too small length") } } Sia-1.0.4/crypto/merkle.go000066400000000000000000000107721302033070500153560ustar00rootroot00000000000000package crypto import ( "bytes" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/merkletree" ) const ( // SegmentSize is the chunk size that is used when taking the Merkle root // of a file. 64 is chosen because bandwidth is scarce and it optimizes for // the smallest possible storage proofs. Using a larger base, even 256 // bytes, would result in substantially faster hashing, but the bandwidth // tradeoff was deemed to be more important, as blockchain space is scarce. SegmentSize = 64 ) // MerkleTree wraps merkletree.Tree, changing some of the function definitions // to assume sia-specific constants and return sia-specific types. type MerkleTree struct { merkletree.Tree } // NewTree returns a MerkleTree, which can be used for getting Merkle roots and // Merkle proofs on data. See merkletree.Tree for more details. func NewTree() *MerkleTree { return &MerkleTree{*merkletree.New(NewHash())} } // PushObject encodes and adds the hash of the encoded object to the tree as a // leaf. func (t *MerkleTree) PushObject(obj interface{}) { t.Push(encoding.Marshal(obj)) } // Root is a redefinition of merkletree.Tree.Root, returning a Hash instead of // a []byte. func (t *MerkleTree) Root() (h Hash) { copy(h[:], t.Tree.Root()) return } // CachedMerkleTree wraps merkletree.CachedTree, changing some of the function // definitions to assume sia-specific constants and return sia-specific types. type CachedMerkleTree struct { merkletree.CachedTree } // NewCachedTree returns a CachedMerkleTree, which can be used for getting // Merkle roots and proofs from data that has cached subroots. See // merkletree.CachedTree for more details. func NewCachedTree(height uint64) *CachedMerkleTree { return &CachedMerkleTree{*merkletree.NewCachedTree(NewHash(), height)} } // Prove is a redefinition of merkletree.CachedTree.Prove, so that Sia-specific // types are used instead of the generic types used by the parent package. The // base is not a return value because the base is used as input. func (ct *CachedMerkleTree) Prove(base []byte, cachedHashSet []Hash) []Hash { // Turn the input in to a proof set that will be recognized by the high // level tree. cachedProofSet := make([][]byte, len(cachedHashSet)+1) cachedProofSet[0] = base for i := range cachedHashSet { cachedProofSet[i+1] = cachedHashSet[i][:] } _, proofSet, _, _ := ct.CachedTree.Prove(cachedProofSet) // convert proofSet to base and hashSet hashSet := make([]Hash, len(proofSet)-1) for i, proof := range proofSet[1:] { copy(hashSet[i][:], proof) } return hashSet } // Push is a redefinition of merkletree.CachedTree.Push, with the added type // safety of only accepting a hash. func (ct *CachedMerkleTree) Push(h Hash) { ct.CachedTree.Push(h[:]) } // Root is a redefinition of merkletree.CachedTree.Root, returning a Hash // instead of a []byte. func (ct *CachedMerkleTree) Root() (h Hash) { copy(h[:], ct.CachedTree.Root()) return } // CalculateLeaves calculates the number of leaves that would be pushed from // data of size 'dataSize'. func CalculateLeaves(dataSize uint64) uint64 { numSegments := dataSize / SegmentSize if dataSize == 0 || dataSize%SegmentSize != 0 { numSegments++ } return numSegments } // MerkleRoot returns the Merkle root of the input data. func MerkleRoot(b []byte) Hash { t := NewTree() buf := bytes.NewBuffer(b) for buf.Len() > 0 { t.Push(buf.Next(SegmentSize)) } return t.Root() } // MerkleProof builds a Merkle proof that the data at segment 'proofIndex' is a // part of the Merkle root formed by 'b'. func MerkleProof(b []byte, proofIndex uint64) (base []byte, hashSet []Hash) { // Create the tree. t := NewTree() t.SetIndex(proofIndex) // Fill the tree. buf := bytes.NewBuffer(b) for buf.Len() > 0 { t.Push(buf.Next(SegmentSize)) } // Get the proof and convert it to a base + hash set. _, proof, _, _ := t.Prove() if len(proof) == 0 { // There's no proof, because there's no data. Return blank values. return nil, nil } base = proof[0] hashSet = make([]Hash, len(proof)-1) for i, p := range proof[1:] { copy(hashSet[i][:], p) } return base, hashSet } // VerifySegment will verify that a segment, given the proof, is a part of a // Merkle root. func VerifySegment(base []byte, hashSet []Hash, numSegments, proofIndex uint64, root Hash) bool { // convert base and hashSet to proofSet proofSet := make([][]byte, len(hashSet)+1) proofSet[0] = base for i := range hashSet { proofSet[i+1] = hashSet[i][:] } return merkletree.VerifyProof(NewHash(), root[:], proofSet, proofIndex, numSegments) } Sia-1.0.4/crypto/merkle_test.go000066400000000000000000000115751302033070500164170ustar00rootroot00000000000000package crypto import ( "crypto/rand" "testing" ) // TestTreeBuilder builds a tree and gets the merkle root. func TestTreeBuilder(t *testing.T) { tree := NewTree() tree.PushObject("a") tree.PushObject("b") _ = tree.Root() // Correctness is assumed, as it's tested by the merkletree package. This // function is really for code coverage. } // TestCalculateLeaves probes the CalculateLeaves function. func TestCalculateLeaves(t *testing.T) { tests := []struct { size, expSegs uint64 }{ {0, 1}, {63, 1}, {64, 1}, {65, 2}, {127, 2}, {128, 2}, {129, 3}, } for i, test := range tests { if segs := CalculateLeaves(test.size); segs != test.expSegs { t.Errorf("miscalculation for test %v: expected %v, got %v", i, test.expSegs, segs) } } } // TestStorageProof builds a storage proof and checks that it verifies // correctly. func TestStorageProof(t *testing.T) { // Generate proof data. numSegments := uint64(7) data := make([]byte, numSegments*SegmentSize) rand.Read(data) rootHash := MerkleRoot(data) // Create and verify proofs for all indices. for i := uint64(0); i < numSegments; i++ { baseSegment, hashSet := MerkleProof(data, i) if !VerifySegment(baseSegment, hashSet, numSegments, i, rootHash) { t.Error("Proof", i, "did not pass verification") } } // Try an incorrect proof. baseSegment, hashSet := MerkleProof(data, 3) if VerifySegment(baseSegment, hashSet, numSegments, 4, rootHash) { t.Error("Verified a bad proof") } } // TestNonMultipleNumberOfSegmentsStorageProof builds a storage proof that has // a last leaf of size less than SegmentSize. func TestNonMultipleLeafSizeStorageProof(t *testing.T) { // Generate proof data. data := make([]byte, (2*SegmentSize)+10) rand.Read(data) rootHash := MerkleRoot(data) // Create and verify a proof for the last index. baseSegment, hashSet := MerkleProof(data, 2) if !VerifySegment(baseSegment, hashSet, 3, 2, rootHash) { t.Error("padded segment proof failed") } } // TestCachedTree tests the cached tree functions of the package. func TestCachedTree(t *testing.T) { if testing.Short() { t.SkipNow() } // Build a cached tree out of 4 subtrees, each subtree of height 2 (4 // elements). tree1Bytes, err := RandBytes(SegmentSize * 4) if err != nil { t.Fatal(err) } tree2Bytes, err := RandBytes(SegmentSize * 4) if err != nil { t.Fatal(err) } tree3Bytes, err := RandBytes(SegmentSize * 4) if err != nil { t.Fatal(err) } tree4Bytes, err := RandBytes(SegmentSize * 4) if err != nil { t.Fatal(err) } tree1Root := MerkleRoot(tree1Bytes) tree2Root := MerkleRoot(tree2Bytes) tree3Root := MerkleRoot(tree3Bytes) tree4Root := MerkleRoot(tree4Bytes) fullRoot := MerkleRoot(append(tree1Bytes, append(tree2Bytes, append(tree3Bytes, tree4Bytes...)...)...)) // Get a cached proof for index 0. base, cachedHashSet := MerkleProof(tree1Bytes, 0) if !VerifySegment(base, cachedHashSet, 4, 0, tree1Root) { t.Fatal("the proof for the subtree was invalid") } ct := NewCachedTree(2) ct.SetIndex(0) ct.Push(tree1Root) ct.Push(tree2Root) ct.Push(tree3Root) ct.Push(tree4Root) hashSet := ct.Prove(base, cachedHashSet) if !VerifySegment(base, hashSet, 4*4, 0, fullRoot) { t.Fatal("cached proof construction appears unsuccessful") } if ct.Root() != fullRoot { t.Fatal("cached Merkle root is not matching the full Merkle root") } // Get a cached proof for index 6. base, cachedHashSet = MerkleProof(tree2Bytes, 2) if !VerifySegment(base, cachedHashSet, 4, 2, tree2Root) { t.Fatal("the proof for the subtree was invalid") } ct = NewCachedTree(2) ct.SetIndex(6) ct.Push(tree1Root) ct.Push(tree2Root) ct.Push(tree3Root) ct.Push(tree4Root) hashSet = ct.Prove(base, cachedHashSet) if !VerifySegment(base, hashSet, 4*4, 6, fullRoot) { t.Fatal("cached proof construction appears unsuccessful") } if ct.Root() != fullRoot { t.Fatal("cached Merkle root is not matching the full Merkle root") } } // TestMerkleTreeOddDataSize checks that MerkleRoot and MerkleProof still // function correctly if you provide data which does not have a size evenly // divisible by SegmentSize. func TestOddDataSize(t *testing.T) { if testing.Short() { t.SkipNow() } // Create some random data that's not evenly padded. for i := 0; i < 25; i++ { randFullSegments, err := RandIntn(65) if err != nil { t.Fatal(err) } randOverflow, err := RandIntn(63) if err != nil { t.Fatal(err) } randOverflow++ // Range is [1, 63] instead of [0, 62] randProofIndex, err := RandIntn(randFullSegments + 1) if err != nil { t.Fatal(err) } data, err := RandBytes(SegmentSize*randFullSegments + randOverflow) if err != nil { t.Fatal(err) } root := MerkleRoot(data) base, hashSet := MerkleProof(data, uint64(randProofIndex)) if !VerifySegment(base, hashSet, uint64(randFullSegments)+1, uint64(randProofIndex), root) { t.Error("Padded data proof failed for", randFullSegments, randOverflow, randProofIndex) } } } Sia-1.0.4/crypto/rand.go000066400000000000000000000012761302033070500150220ustar00rootroot00000000000000package crypto import ( "crypto/rand" "math/big" ) // RandBytes returns n bytes of random data. func RandBytes(n int) ([]byte, error) { b := make([]byte, n) _, err := rand.Read(b) return b, err } // RandIntn returns a non-negative random integer in the range [0,n). It panics // if n <= 0. func RandIntn(n int) (int, error) { r, err := rand.Int(rand.Reader, big.NewInt(int64(n))) return int(r.Int64()), err } // Perm returns, as a slice of n ints, a random permutation of the integers // [0,n). func Perm(n int) ([]int, error) { m := make([]int, n) for i := 0; i < n; i++ { j, err := RandIntn(i + 1) if err != nil { return nil, err } m[i] = m[j] m[j] = i } return m, nil } Sia-1.0.4/crypto/rand_test.go000066400000000000000000000023761302033070500160630ustar00rootroot00000000000000package crypto import ( "testing" ) // panics returns true if the function fn panicked. func panics(fn func()) (panicked bool) { defer func() { panicked = (recover() != nil) }() fn() return } // TestRandIntnPanics tests that RandIntn panics if n <= 0. func TestRandIntnPanics(t *testing.T) { // Test n = 0. if !panics(func() { RandIntn(0) }) { t.Error("expected panic for n <= 0") } // Test n < 0. if !panics(func() { RandIntn(-1) }) { t.Error("expected panic for n <= 0") } } // TestPerm tests the Perm function. func TestPerm(t *testing.T) { chars := "abcde" // string to be permuted createPerm := func() string { perm, err := Perm(len(chars)) if err != nil { t.Fatal(err) } s := make([]byte, len(chars)) for i, j := range perm { s[i] = chars[j] } return string(s) } // create (factorial(len(chars)) * 100) permutations permCount := make(map[string]int) for i := 0; i < 12000; i++ { permCount[createPerm()]++ } // we should have seen each permutation approx. 100 times for p, n := range permCount { if n < 50 || n > 150 { t.Errorf("saw permutation %v times: %v", n, p) } } } func BenchmarkRandIntn(b *testing.B) { for n := 0; n < b.N; n++ { _, err := RandIntn(4500) if err != nil { b.Fatal(err) } } } Sia-1.0.4/crypto/signatures.go000066400000000000000000000064421302033070500162620ustar00rootroot00000000000000package crypto import ( "errors" "io" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/ed25519" ) const ( // EntropySize defines the amount of entropy necessary to do secure // cryptographic operations, in bytes. EntropySize = ed25519.EntropySize // PublicKeySize defines the size of public keys in bytes. PublicKeySize = ed25519.PublicKeySize // SecretKeySize defines the size of secret keys in bytes. SecretKeySize = ed25519.SecretKeySize // SignatureSize defines the size of signatures in bytes. SignatureSize = ed25519.SignatureSize ) var ( // ErrInvalidSignature is returned if a signature is provided that does not // match the data and public key. ErrInvalidSignature = errors.New("invalid signature") ) type ( // PublicKey is an object that can be used to verify signatures. PublicKey [PublicKeySize]byte // SecretKey can be used to sign data for the corresponding public key. SecretKey [SecretKeySize]byte // Signature proves that data was signed by the owner of a particular // public key's corresponding secret key. Signature [SignatureSize]byte ) // PublicKey returns the public key that corresponds to a secret key. func (sk SecretKey) PublicKey() (pk PublicKey) { copy(pk[:], sk[SecretKeySize-PublicKeySize:]) return } // GenerateKeyPair creates a public-secret keypair that can be used to sign and verify // messages. func GenerateKeyPair() (sk SecretKey, pk PublicKey, err error) { return stdKeyGen.generate() } // GenerateKeyPairDeterministic generates keys deterministically using the input // entropy. The input entropy must be 32 bytes in length. func GenerateKeyPairDeterministic(entropy [EntropySize]byte) (SecretKey, PublicKey) { return stdKeyGen.generateDeterministic(entropy) } // ReadSignedObject reads a length-prefixed object prefixed by its signature, // and verifies the signature. func ReadSignedObject(r io.Reader, obj interface{}, maxLen uint64, pk PublicKey) error { // read the signature var sig Signature err := encoding.NewDecoder(r).Decode(&sig) if err != nil { return err } // read the encoded object encObj, err := encoding.ReadPrefix(r, maxLen) if err != nil { return err } // verify the signature if err := VerifyHash(HashBytes(encObj), pk, sig); err != nil { return err } // decode the object return encoding.Unmarshal(encObj, obj) } // SignHash signs a message using a secret key. Though the current // implementation will never return an error, switching libraries in the future // may result in errors that can be returned. func SignHash(data Hash, sk SecretKey) (sig Signature, err error) { skNorm := [SecretKeySize]byte(sk) sig = *ed25519.Sign(&skNorm, data[:]) return sig, nil } // VerifyHash uses a public key and input data to verify a signature. func VerifyHash(data Hash, pk PublicKey, sig Signature) error { pkNorm := [PublicKeySize]byte(pk) sigNorm := [SignatureSize]byte(sig) verifies := ed25519.Verify(&pkNorm, data[:], &sigNorm) if !verifies { return ErrInvalidSignature } return nil } // WriteSignedObject writes a length-prefixed object prefixed by its signature. func WriteSignedObject(w io.Writer, obj interface{}, sk SecretKey) error { objBytes := encoding.Marshal(obj) sig, err := SignHash(HashBytes(objBytes), sk) if err != nil { return err } return encoding.NewEncoder(w).EncodeAll(sig, objBytes) } Sia-1.0.4/crypto/signatures_mock.go000066400000000000000000000032151302033070500172660ustar00rootroot00000000000000package crypto import ( "crypto/rand" "io" "github.com/NebulousLabs/ed25519" ) type ( // keyDeriver contains all of the dependencies for a signature key // generator. The dependencies are separated to enable mocking. keyDeriver interface { deriveKeyPair([EntropySize]byte) (ed25519.SecretKey, ed25519.PublicKey) } ) var ( // stdKeyGen is a signature generator that can be used to generate random // and deterministic keys for signing objects. stdKeyGen sigKeyGen = sigKeyGen{entropySource: rand.Reader, keyDeriver: &stdKeyDeriver{}} ) // sigKeyGen contains a set of dependencies that are used to build out the core // logic for generating keys in Sia. type sigKeyGen struct { entropySource io.Reader keyDeriver keyDeriver } // generate builds a signature keypair using a sigKeyGen to manage // dependencies. func (skg sigKeyGen) generate() (sk SecretKey, pk PublicKey, err error) { var entropy [EntropySize]byte _, err = io.ReadFull(skg.entropySource, entropy[:]) if err != nil { return } skPointer, pkPointer := skg.keyDeriver.deriveKeyPair(entropy) return *skPointer, *pkPointer, nil } // generateDeterministic builds a signature keypair deterministically using a // sigKeyGen to manage dependencies. func (skg sigKeyGen) generateDeterministic(entropy [EntropySize]byte) (SecretKey, PublicKey) { skPointer, pkPointer := skg.keyDeriver.deriveKeyPair(entropy) return *skPointer, *pkPointer } // stdKeyDeriver implements the keyDeriver dependency for the sigKeyGen. type stdKeyDeriver struct{} func (skd *stdKeyDeriver) deriveKeyPair(entropy [EntropySize]byte) (ed25519.SecretKey, ed25519.PublicKey) { return ed25519.GenerateKey(entropy) } Sia-1.0.4/crypto/signatures_test.go000066400000000000000000000217231302033070500173200ustar00rootroot00000000000000package crypto import ( "bytes" "crypto/rand" "errors" "testing" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/ed25519" ) // mockKeyDeriver is a mock implementation of keyDeriver that saves its provided // entropy and allows the client to specify the returned SecretKey and // PublicKey. type mockKeyDeriver struct { called bool entropy [EntropySize]byte sk ed25519.SecretKey pk ed25519.PublicKey } // deriveKeyPair is a mocked key deriver operating on the mockKeyDeriver // struct. func (kd *mockKeyDeriver) deriveKeyPair(entropy [EntropySize]byte) (ed25519.SecretKey, ed25519.PublicKey) { kd.called = true kd.entropy = entropy return kd.sk, kd.pk } // TestUnitGenerateRandomKeyPair checks that the Generate method is properly // calling its dependencies and returning the expected key pair. func TestUnitGenerateRandomKeyPair(t *testing.T) { var mockEntropy [EntropySize]byte mockEntropy[0] = 5 mockEntropy[EntropySize-1] = 5 entropyReader := bytes.NewReader(mockEntropy[:]) sk := ed25519.SecretKey(&[SecretKeySize]byte{}) sk[0] = 7 sk[32] = 8 pk := ed25519.PublicKey(&[PublicKeySize]byte{}) pk[0] = sk[32] kd := mockKeyDeriver{sk: sk, pk: pk} // Create a SignatureKeyGenerator using mocks. g := sigKeyGen{entropyReader, &kd} // Create key pair. skActual, pkActual, err := g.generate() // Verify that we got back the expected results. if err != nil { t.Error(err) } if *sk != skActual { t.Errorf("Generated secret key does not match expected! expected = %v, actual = %v", sk, skActual) } if *pk != pkActual { t.Errorf("Generated public key does not match expected! expected = %v, actual = %v", pk, pkActual) } // Verify the dependencies were called correctly if !kd.called { t.Error("keyDeriver was never called.") } if mockEntropy != kd.entropy { t.Errorf("keyDeriver was called with the wrong entropy. expected = %v, actual = %v", mockEntropy, kd.entropy) } } // failingReader is a mock implementation of io.Reader that fails with a client- // defined error. type failingReader struct { err error } // Read satisfies the entropySource of the stdKeyGen, but always fails, // resulting in errors that must be handled by the generate function. func (fr failingReader) Read([]byte) (int, error) { return 0, fr.err } // TestUnitGenerateRandomKeyBadRand checks that the Generate method fails if // the call to entropy source fails func TestUnitGenerateRandomKeyBadRand(t *testing.T) { fr := failingReader{err: errors.New("mock error from entropy reader")} g := sigKeyGen{entropySource: &fr} if _, _, err := g.generate(); err == nil { t.Error("Generate should fail when entropy source fails.") } } // TestUnitGenerateDeterministicKeyPari checks that the GenerateDeterministic // method is properly calling its dependencies and returning the expected key // pair. func TestUnitGenerateDeterministicKeyPair(t *testing.T) { // Create entropy bytes, setting a few bytes explicitly instead of using a // buffer of random bytes. var mockEntropy [EntropySize]byte mockEntropy[0] = 4 mockEntropy[EntropySize-1] = 5 sk := ed25519.SecretKey(&[SecretKeySize]byte{}) sk[0] = 7 sk[32] = 8 pk := ed25519.PublicKey(&[PublicKeySize]byte{}) pk[0] = sk[32] kd := mockKeyDeriver{sk: sk, pk: pk} g := sigKeyGen{keyDeriver: &kd} // Create key pair. skActual, pkActual := g.generateDeterministic(mockEntropy) // Verify that we got back the right results. if *sk != skActual { t.Errorf("Generated secret key does not match expected! expected = %v, actual = %v", sk, skActual) } if *pk != pkActual { t.Errorf("Generated public key does not match expected! expected = %v, actual = %v", pk, pkActual) } // Verify the dependencies were called correctly. if !kd.called { t.Error("keyDeriver was never called.") } if mockEntropy != kd.entropy { t.Errorf("keyDeriver was called with the wrong entropy. expected = %v, actual = %v", mockEntropy, kd.entropy) } } // TestUnitSignatureEncoding creates and encodes a public key, and verifies // that it decodes correctly, does the same with a signature. func TestUnitSignatureEncoding(t *testing.T) { // Create a dummy key pair. var sk SecretKey sk[0] = 4 sk[32] = 5 pk := sk.PublicKey() // Marshal and unmarshal the public key. marshalledPK := encoding.Marshal(pk) var unmarshalledPK PublicKey err := encoding.Unmarshal(marshalledPK, &unmarshalledPK) if err != nil { t.Fatal(err) } // Test the public keys for equality. if pk != unmarshalledPK { t.Error("pubkey not the same after marshalling and unmarshalling") } // Create a signature using the secret key. var signedData Hash rand.Read(signedData[:]) sig, err := SignHash(signedData, sk) if err != nil { t.Fatal(err) } // Marshal and unmarshal the signature. marshalledSig := encoding.Marshal(sig) var unmarshalledSig Signature err = encoding.Unmarshal(marshalledSig, &unmarshalledSig) if err != nil { t.Fatal(err) } // Test signatures for equality. if sig != unmarshalledSig { t.Error("signature not same after marshalling and unmarshalling") } } // TestUnitSigning creates a bunch of keypairs and signs random data with each of // them. func TestUnitSigning(t *testing.T) { if testing.Short() { t.SkipNow() } // Try a bunch of signatures because at one point there was a library that // worked around 98% of the time. Tests would usually pass, but 200 // iterations would normally cause a failure. iterations := 200 for i := 0; i < iterations; i++ { // Create dummy key pair. sk, pk, err := GenerateKeyPair() if err != nil { t.Fatal(err) } // Generate and sign the data. var randData Hash rand.Read(randData[:]) sig, err := SignHash(randData, sk) if err != nil { t.Fatal(err) } // Verify the signature. err = VerifyHash(randData, pk, sig) if err != nil { t.Fatal(err) } // Attempt to verify after the data has been altered. randData[0] += 1 err = VerifyHash(randData, pk, sig) if err != ErrInvalidSignature { t.Fatal(err) } // Restore the data and make sure the signature is valid again. randData[0] -= 1 err = VerifyHash(randData, pk, sig) if err != nil { t.Fatal(err) } // Attempt to verify after the signature has been altered. sig[0] += 1 err = VerifyHash(randData, pk, sig) if err != ErrInvalidSignature { t.Fatal(err) } } } // TestIntegrationSigKeyGenerate is an integration test checking that // GenerateKeyPair and GenerateKeyPairDeterminisitc accurately create keys. func TestIntegrationSigKeyGeneration(t *testing.T) { if testing.Short() { t.SkipNow() } message := HashBytes([]byte{'m', 's', 'g'}) // Create a random key and use it. randSecKey, randPubKey, err := GenerateKeyPair() if err != nil { t.Fatal(err) } sig, err := SignHash(message, randSecKey) if err != nil { t.Fatal(err) } err = VerifyHash(message, randPubKey, sig) if err != nil { t.Error(err) } // Corrupt the signature sig[0]++ err = VerifyHash(message, randPubKey, sig) if err == nil { t.Error("corruption failed") } // Create a deterministic key and use it. var detEntropy [EntropySize]byte detEntropy[0] = 35 detSecKey, detPubKey := GenerateKeyPairDeterministic(detEntropy) sig, err = SignHash(message, detSecKey) if err != nil { t.Fatal(err) } err = VerifyHash(message, detPubKey, sig) if err != nil { t.Error(err) } // Corrupt the signature sig[0]++ err = VerifyHash(message, detPubKey, sig) if err == nil { t.Error("corruption failed") } } // TestReadWriteSignedObject tests the ReadSignObject and WriteSignedObject // functions, which are inverses of each other. func TestReadWriteSignedObject(t *testing.T) { sk, pk, err := GenerateKeyPair() if err != nil { t.Fatal(err) } // Write signed object into buffer. b := new(bytes.Buffer) err = WriteSignedObject(b, "foo", sk) if err != nil { t.Fatal(err) } // Keep a copy of b's bytes. buf := b.Bytes() // Read and verify object. var read string err = ReadSignedObject(b, &read, 11, pk) if err != nil { t.Fatal(err) } if read != "foo" { t.Fatal("encode/decode mismatch: expected 'foo', got", []byte(read)) } // Check that maxlen is being respected. b = bytes.NewBuffer(buf) // reset b err = ReadSignedObject(b, &read, 10, pk) if err == nil || err.Error() != "length 11 exceeds maxLen of 10" { t.Fatal("expected length error, got", err) } // Disrupt the decoding to get coverage on the failure branch. err = ReadSignedObject(b, &read, 11, pk) if err == nil || err.Error() != "could not decode type crypto.Signature: unexpected EOF" { t.Fatal(err) } // Try with an invalid signature. buf[0]++ // alter the first byte of the signature, invalidating it. b = bytes.NewBuffer(buf) // reset b err = ReadSignedObject(b, &read, 11, pk) if err != ErrInvalidSignature { t.Fatal(err) } } // TestUnitPublicKey tests the PublicKey method func TestUnitPublicKey(t *testing.T) { for i := 0; i < 1000; i++ { sk, pk, err := GenerateKeyPair() if err != nil { t.Fatal(err) } if sk.PublicKey() != pk { t.Error("PublicKey does not match actual public key:", pk, sk.PublicKey()) } } } Sia-1.0.4/doc/000077500000000000000000000000001302033070500127665ustar00rootroot00000000000000Sia-1.0.4/doc/API.md000066400000000000000000001073551302033070500137340ustar00rootroot00000000000000Siad API ======== Sia uses semantic versioning and is backwards compatible to version v1.0.0. API calls return either JSON or no content. Success is indicated by 2xx HTTP status codes, while errors are indicated by 4xx and 5xx HTTP status codes. If an endpoint does not specify its expected status code refer to [#standard-responses](#standard-responses). There may be functional API calls which are not documented. These are not guaranteed to be supported beyond the current release, and should not be used in production. Notes: - Requests must set their User-Agent string to contain the substring "Sia-Agent". - By default, siad listens on "localhost:9980". This can be changed using the `--api-addr` flag when running siad. - **Do not bind or expose the API to a non-loopback address unless you are aware of the possible dangers.** Example GET curl call: ``` curl -A "Sia-Agent" "localhost:9980/wallet/transactions?startheight=1&endheight=250" ``` Example POST curl call: ``` curl -A "Sia-Agent" --data "amount=123&destination=abcd" "localhost:9980/wallet/siacoins" ``` Standard responses ------------------ #### Success The standard response indicating the request was successfully processed is HTTP status code `204 No Content`. If the request was successfully processed and the server responded with JSON the HTTP status code is `200 OK`. Specific endpoints may specify other 2xx status codes on success. #### Error The standard error response indicating the request failed for any reason, is a 4xx or 5xx HTTP status code with an error JSON object describing the error. ```javascript { "message": String // There may be additional fields depending on the specific error. } ``` Authentication -------------- API authentication can be enabled with the `--authenticate-api` siad flag. Authentication is HTTP Basic Authentication as described in [RFC 2617](https://tools.ietf.org/html/rfc2617), however, the username is the empty string. The flag does not enforce authentication on all API endpoints. Only endpoints that expose sensitive information or modify state require authentication. For example, if the API password is "foobar" the request header should include ``` Authorization: Basic OmZvb2Jhcg== ``` Units ----- Unless otherwise specified, all parameters should be specified in their smallest possible unit. For example, size should always be specified in bytes and SiaCoins should be specified in hastings. JSON values returned by the API will also use the smallest possible unit, unless otherwise specified. If a numbers is returned as a string in JSON, it should be treated as an arbitrary-precision number (bignum), and it should be parsed with your language's corresponding bignum library. Currency values are the most common example where this is necessary. Table of contents ----------------- - [Daemon](#daemon) - [Consensus](#consensus) - [Gateway](#gateway) - [Host](#host) - [Host DB](#host-db) - [Miner](#miner) - [Renter](#renter) - [Wallet](#wallet) Daemon ------ | Route | HTTP verb | | ----------------------------------------- | --------- | | [/daemon/constants](#daemonconstants-get) | GET | | [/daemon/stop](#daemonstop-get) | GET | | [/daemon/version](#daemonversion-get) | GET | For examples and detailed descriptions of request and response parameters, refer to [Daemon.md](/doc/api/Daemon.md). #### /daemon/constants [GET] returns the set of constants in use. ###### JSON Response [(with comments)](/doc/api/Daemon.md#json-response) ```javascript { "genesistimestamp": 1257894000, // Unix time "blocksizelimit": 2000000, // bytes "blockfrequency": 600, // seconds per block "targetwindow": 1000, // blocks "mediantimestampwindow": 11, // blocks "futurethreshold": 10800, // seconds "siafundcount": "10000", "siafundportion": "39/1000", "maturitydelay": 144, // blocks "initialcoinbase": 300000, // SiaCoins (see note in Daemon.md) "minimumcoinbase": 30000, // SiaCoins (see note in Daemon.md) "roottarget": [0,0,0,0,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0], "rootdepth": [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255], "maxadjustmentup": "5/2", "maxadjustmentdown": "2/5", "siacoinprecision": "1000000000000000000000000" // hastings per siacoin } ``` #### /daemon/stop [GET] cleanly shuts down the daemon. May take a few seconds. ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /daemon/version [GET] returns the version of the Sia daemon currently running. ###### JSON Response [(with comments)](/doc/api/Daemon.md#json-response-1) ```javascript { "version": "1.0.0" } ``` Consensus --------- | Route | HTTP verb | | ---------------------------- | --------- | | [/consensus](#consensus-get) | GET | For examples and detailed descriptions of request and response parameters, refer to [Consensus.md](/doc/api/Consensus.md). #### /consensus [GET] returns information about the consensus set, such as the current block height. ###### JSON Response [(with comments)](/doc/api/Consensus.md#json-response) ```javascript { "synced": true, "height": 62248, "currentblock": "00000000000008a84884ba827bdc868a17ba9c14011de33ff763bd95779a9cf1", "target": [0,0,0,0,0,0,11,48,125,79,116,89,136,74,42,27,5,14,10,31,23,53,226,238,202,219,5,204,38,32,59,165] } ``` Gateway ------- | Route | HTTP verb | | ---------------------------------------------------------------------------------- | --------- | | [/gateway](#gateway-get-example) | GET | | [/gateway/connect/___:netaddress___](#gatewayconnectnetaddress-post-example) | POST | | [/gateway/disconnect/___:netaddress___](#gatewaydisconnectnetaddress-post-example) | POST | For examples and detailed descriptions of request and response parameters, refer to [Gateway.md](/doc/api/Gateway.md). #### /gateway [GET] [(example)](/doc/api/Gateway.md#gateway-info) returns information about the gateway, including the list of connected peers. ###### JSON Response [(with comments)](/doc/api/Gateway.md#json-response) ```javascript { "netaddress": String, "peers": []{ "netaddress": String, "version": String, "inbound": Boolean } } ``` #### /gateway/connect/___:netaddress___ [POST] [(example)](/doc/api/Gateway.md#connecting-to-a-peer) connects the gateway to a peer. The peer is added to the node list if it is not already present. The node list is the list of all nodes the gateway knows about, but is not necessarily connected to. ###### Path Parameters [(with comments)](/doc/api/Gateway.md#path-parameters) ``` :netaddress ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /gateway/disconnect/___:netaddress___ [POST] [(example)](/doc/api/Gateway.md#disconnecting-from-a-peer) disconnects the gateway from a peer. The peer remains in the node list. ###### Path Parameters [(with comments)](/doc/api/Gateway.md#path-parameters-1) ``` :netaddress ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). Host ---- | Route | HTTP verb | | ------------------------------------------------------------------------------------- | --------- | | [/host](#host-get) | GET | | [/host](#host-post) | POST | | [/host/announce](#hostannounce-post) | POST | | [/host/storage](#hoststorage-get) | GET | | [/host/storage/folders/add](#hoststoragefoldersadd-post) | POST | | [/host/storage/folders/remove](#hoststoragefoldersremove-post) | POST | | [/host/storage/folders/resize](#hoststoragefoldersresize-post) | POST | | [/host/storage/sectors/delete/___:merkleroot___](#hoststoragesectorsdeletemerkleroot) | POST | For examples and detailed descriptions of request and response parameters, refer to [Host.md](/doc/api/Host.md). #### /host [GET] fetches status information about the host. ###### JSON Response [(with comments)](/doc/api/Host.md#json-response) ```javascript { "externalsettings": { "acceptingcontracts": true, "maxdownloadbatchsize": 17825792, // bytes "maxduration": 25920, // blocks "maxrevisebatchsize": 17825792, // bytes "netaddress": "123.456.789.0:9982", "remainingstorage": 35000000000, // bytes "sectorsize": 4194304, // bytes "totalstorage": 35000000000, // bytes "unlockhash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab", "windowsize": 144, // blocks "collateral": "57870370370", // hastings / byte / block "maxcollateral": "100000000000000000000000000000", // hastings "contractprice": "30000000000000000000000000", // hastings "downloadbandwidthprice": "250000000000000", // hastings / byte "storageprice": "231481481481", // hastings / byte / block "uploadbandwidthprice": "100000000000000", // hastings / byte "revisionnumber": 0, "version": "1.0.0" }, "financialmetrics": { "contractcount": 2, "contractcompensation": "123", // hastings "potentialcontractcompensation": "123", // hastings "lockedstoragecollateral": "123", // hastings "lostrevenue": "123", // hastings "loststoragecollateral": "123", // hastings "potentialstoragerevenue": "123", // hastings "riskedstoragecollateral": "123", // hastings "storagerevenue": "123", // hastings "transactionfeeexpenses": "123", // hastings "downloadbandwidthrevenue": "123", // hastings "potentialdownloadbandwidthrevenue": "123", // hastings "potentialuploadbandwidthrevenue": "123", // hastings "uploadbandwidthrevenue": "123" // hastings }, "internalsettings": { "acceptingcontracts": true, "maxdownloadbatchsize": 17825792, // bytes "maxduration": 25920, // blocks "maxrevisebatchsize": 17825792, // bytes "netaddress": "123.456.789.0:9982", "windowsize": 144, // blocks "collateral": "57870370370", // hastings / byte / block "collateralbudget": "2000000000000000000000000000000", // hastings "maxcollateral": "100000000000000000000000000000", // hastings "mincontractprice": "30000000000000000000000000", // hastings "mindownloadbandwidthprice": "250000000000000", // hastings / byte "minstorageprice": "231481481481", // hastings / byte / block "minuploadbandwidthprice": "100000000000000" // hastings / byte }, "networkmetrics": { "downloadcalls": 0, "errorcalls": 1, "formcontractcalls": 2, "renewcalls": 3, "revisecalls": 4, "settingscalls": 5, "unrecognizedcalls": 6 } } ``` #### /host [POST] configures hosting parameters. All parameters are optional; unspecified parameters will be left unchanged. ###### Query String Parameters [(with comments)](/doc/api/Host.md#query-string-parameters) ``` acceptingcontracts // Optional, true / false maxdownloadbatchsize // Optional, bytes maxduration // Optional, blocks maxrevisebatchsize // Optional, bytes netaddress // Optional windowsize // Optional, blocks collateral // Optional, hastings / byte / block collateralbudget // Optional, hastings maxcollateral // Optional, hastings mincontractprice // Optional, hastings mindownloadbandwidthprice // Optional, hastings / byte minstorageprice // Optional, hastings / byte / block minuploadbandwidthprice // Optional, hastings / byte ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /host/announce [POST] Announces the host to the network as a source of storage. Generally only needs to be called once. ###### Query String Parameters [(with comments)](/doc/api/Host.md#query-string-parameters-1) ``` netaddress string // Optional ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /host/storage [GET] gets a list of folders tracked by the host's storage manager. ###### JSON Response [(with comments)](/doc/api/Host.md#json-response-1) ```javascript { "folders": [ { "path": "/home/foo/bar", "capacity": 50000000000, // bytes "capacityremaining": 100000, // bytes "failedreads": 0, "failedwrites": 1, "successfulreads": 2, "successfulwrites": 3 } ] } ``` #### /host/storage/folders/add [POST] adds a storage folder to the manager. The manager may not check that there is enough space available on-disk to support as much storage as requested ###### Query String Parameters [(with comments)](/doc/api/Host.md#query-string-parameters-2) ``` path // Required size // bytes, Required ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /host/storage/folders/remove [POST] remove a storage folder from the manager. All storage on the folder will be moved to other storage folders, meaning that no data will be lost. If the manager is unable to save data, an error will be returned and the operation will be stopped. ###### Query String Parameters [(with comments)](/doc/api/Host.md#query-string-parameters-3) ``` path // Required force // bool, Optional, default is false ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /host/storage/folders/resize [POST] grows or shrink a storage folder in the manager. The manager may not check that there is enough space on-disk to support growing the storage folder, but should gracefully handle running out of space unexpectedly. When shrinking a storage folder, any data in the folder that needs to be moved will be placed into other storage folders, meaning that no data will be lost. If the manager is unable to migrate the data, an error will be returned and the operation will be stopped. ###### Query String Parameters [(with comments)](/doc/api/Host.md#query-string-parameters-4) ``` path // Required newsize // bytes, Required ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /host/storage/sectors/delete/___:merkleroot___ [POST] deletes a sector, meaning that the manager will be unable to upload that sector and be unable to provide a storage proof on that sector. This endpoint is for removing the data entirely, and will remove instances of the sector appearing at all heights. The primary purpose is to comply with legal requests to remove data. ###### Path Parameters [(with comments)](/doc/api/Host.md#path-parameters) ``` :merkleroot ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). Host DB ------- | Request | HTTP Verb | | ------------------------------------------- | --------- | | [/hostdb/active](#hostdbactive-get-example) | GET | | [/hostdb/all](#hostdball-get-example) | GET | For examples and detailed descriptions of request and response parameters, refer to [HostDB.md](/doc/api/HostDB.md). #### /hostdb/active [GET] [(example)](/doc/api/HostDB.md#active-hosts) lists all of the active hosts known to the renter, sorted by preference. ###### Query String Parameters [(with comments)](/doc/api/HostDB.md#query-string-parameters) ``` numhosts // Optional ``` ###### JSON Response [(with comments)](/doc/api/HostDB.md#json-response) ```javascript { "hosts": [ { "acceptingcontracts": true, "maxdownloadbatchsize": 17825792, // bytes "maxduration": 25920, // blocks "maxrevisebatchsize": 17825792, // bytes "netaddress": "123.456.789.2:9982", "remainingstorage": 35000000000, // bytes "sectorsize": 4194304, // bytes "totalstorage": 35000000000, // bytes "unlockhash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab", "windowsize": 144, // blocks "publickey": { "algorithm": "ed25519", "key": "RW50cm9weSBpc24ndCB3aGF0IGl0IHVzZWQgdG8gYmU=" } } ] } ``` #### /hostdb/all [GET] [(example)](/doc/api/HostDB.md#all-hosts) lists all of the hosts known to the renter. Hosts are not guaranteed to be in any particular order, and the order may change in subsequent calls. ###### JSON Response [(with comments)](/doc/api/HostDB.md#json-response-1) ```javascript { "hosts": [ { "acceptingcontracts": true, "maxdownloadbatchsize": 17825792, // bytes "maxduration": 25920, // blocks "maxrevisebatchsize": 17825792, // bytes "netaddress": "123.456.789.0:9982", "remainingstorage": 35000000000, // bytes "sectorsize": 4194304, // bytes "totalstorage": 35000000000, // bytes "unlockhash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab", "windowsize": 144, // blocks "publickey": { "algorithm": "ed25519", "key": "RW50cm9weSBpc24ndCB3aGF0IGl0IHVzZWQgdG8gYmU=" } } ] } ``` Miner ----- | Route | HTTP verb | | ---------------------------------- | --------- | | [/miner](#miner-get) | GET | | [/miner/start](#minerstart-get) | GET | | [/miner/stop](#minerstop-get) | GET | | [/miner/header](#minerheader-get) | GET | | [/miner/header](#minerheader-post) | POST | For examples and detailed descriptions of request and response parameters, refer to [Miner.md](/doc/api/Miner.md). #### /miner [GET] returns the status of the miner. ###### JSON Response [(with comments)](/doc/api/Miner.md#json-response) ```javascript { "blocksmined": 9001, "cpuhashrate": 1337, "cpumining": false, "staleblocksmined": 0, } ``` #### /miner/start [GET] starts a single threaded cpu miner. Does nothing if the cpu miner is already running. ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /miner/stop [GET] stops the cpu miner. Does nothing if the cpu miner is not running. ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /miner/header [GET] provides a block header that is ready to be grinded on for work. ###### Byte Response For efficiency the header for work is returned as a raw byte encoding of the header, rather than encoded to JSON. Refer to [Miner.md#byte-response](/doc/api/Miner.md#byte-response) for a detailed description of the byte encoding. #### /miner/header [POST] submits a header that has passed the POW. ###### Request Body Bytes For efficiency headers are submitted as raw byte encodings of the header in the body of the request, rather than as a query string parameter or path parameter. The request body should contain only the 80 bytes of the encoded header. The encoding is the same encoding used in `/miner/header [GET]` endpoint. Refer to [Miner.md#byte-response](/doc/api/Miner.md#byte-response) for a detailed description of the byte encoding. Renter ------ | Route | HTTP verb | | ------------------------------------------------------------- | --------- | | [/renter](#renter-get) | GET | | [/renter](#renter-post) | POST | | [/renter/contracts](#rentercontracts-get) | GET | | [/renter/downloads](#renterdownloads-get) | GET | | [/renter/files](#renterfiles-get) | GET | | [/renter/delete/___*siapath___](#renterdeletesiapath-post) | POST | | [/renter/download/___*siapath___](#renterdownloadsiapath-get) | GET | | [/renter/rename/___*siapath___](#renterrenamesiapath-post) | POST | | [/renter/upload/___*siapath___](#renteruploadsiapath-post) | POST | For examples and detailed descriptions of request and response parameters, refer to [Renter.md](/doc/api/Renter.md). #### /renter [GET] returns the current settings along with metrics on the renter's spending. ###### JSON Response [(with comments)](/doc/api/Renter.md#json-response) ```javascript { "settings": { "allowance": { "funds": "1234", // hastings "hosts": 24, "period": 6048, // blocks "renewwindow": 3024 // blocks } }, "financialmetrics": { "contractspending": "1234", // hastings "downloadspending": "5678", // hastings "storagespending": "1234", // hastings "uploadspending": "5678" // hastings } } ``` #### /renter [POST] modify settings that control the renter's behavior. ###### Query String Parameters [(with comments)](/doc/api/Renter.md#query-string-parameters) ``` funds // hastings period // block height ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /renter/contracts [GET] returns active contracts. Expired contracts are not included. ###### JSON Response [(with comments)](/doc/api/Renter.md#json-response-1) ```javascript { "contracts": [ { "endheight": 50000, // block height "id": "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "netaddress": "12.34.56.78:9", "renterfunds": "1234", // hastings "size": 8192 // bytes } ] } ``` #### /renter/downloads [GET] lists all files in the download queue. ###### JSON Response [(with comments)](/doc/api/Renter.md#json-response-2) ```javascript { "downloads": [ { "siapath": "foo/bar.txt", "destination": "/home/users/alice/bar.txt", "filesize": 8192, // bytes "received": 4096, // bytes "starttime": "2009-11-10T23:00:00Z" // RFC 3339 time } ] } ``` #### /renter/files [GET] lists the status of all files. ###### JSON Response [(with comments)](/doc/api/Renter.md#json-response-3) ```javascript { "files": [ { "siapath": "foo/bar.txt", "filesize": 8192, // bytes "available": true, "renewing": true, "redundancy": 5, "uploadprogress": 100, // percent "expiration": 60000 } ] } ``` #### /renter/delete/___*siapath___ [POST] deletes a renter file entry. Does not delete any downloads or original files, only the entry in the renter. ###### Path Parameters [(with comments)](/doc/api/Renter.md#path-parameters) ``` *siapath ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /renter/download/___*siapath___ [GET] downloads a file to the local filesystem. The call will block until the file has been downloaded. ###### Path Parameters [(with comments)](/doc/api/Renter.md#path-parameters-1) ``` *siapath ``` ###### Query String Parameters [(with comments)](/doc/api/Renter.md#query-string-parameters-1) ``` destination ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /renter/rename/___*siapath___ [POST] renames a file. Does not rename any downloads or source files, only renames the entry in the renter. An error is returned if `siapath` does not exist or `newsiapath` already exists. ###### Path Parameters [(with comments)](/doc/api/Renter.md#path-parameters-2) ``` *siapath ``` ###### Query String Parameters [(with comments)](/doc/api/Renter.md#query-string-parameters-2) ``` newsiapath ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /renter/upload/___*siapath___ [POST] uploads a file to the network from the local filesystem. ###### Path Parameters [(with comments)](/doc/api/Renter.md#path-parameters-3) ``` *siapath ``` ###### Query String Parameters [(with comments)](/doc/api/Renter.md#query-string-parameters-2) ``` source ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). Wallet ------ | Route | HTTP verb | | --------------------------------------------------------------- | --------- | | [/wallet](#wallet-get) | GET | | [/wallet/033x](#wallet033x-post) | POST | | [/wallet/address](#walletaddress-get) | GET | | [/wallet/addresses](#walletaddresses-get) | GET | | [/wallet/backup](#walletbackup-get) | GET | | [/wallet/init](#walletinit-post) | POST | | [/wallet/lock](#walletlock-post) | POST | | [/wallet/seed](#walletseed-post) | POST | | [/wallet/seeds](#walletseeds-get) | GET | | [/wallet/siacoins](#walletsiacoins-post) | POST | | [/wallet/siafunds](#walletsiafunds-post) | POST | | [/wallet/siagkey](#walletsiagkey-post) | POST | | [/wallet/transaction/___:id___](#wallettransactionid-get) | GET | | [/wallet/transactions](#wallettransactions-get) | GET | | [/wallet/transactions/___:addr___](#wallettransactionsaddr-get) | GET | | [/wallet/unlock](#walletunlock-post) | POST | For examples and detailed descriptions of request and response parameters, refer to [Wallet.md](/doc/api/Wallet.md). #### /wallet [GET] returns basic information about the wallet, such as whether the wallet is locked or unlocked. ###### JSON Response [(with comments)](/doc/api/Wallet.md#json-response) ```javascript { "encrypted": true, "unlocked": true, "confirmedsiacoinbalance": "123456", // hastings, big int "unconfirmedoutgoingsiacoins": "0", // hastings, big int "unconfirmedincomingsiacoins": "789", // hastings, big int "siafundbalance": "1", // siafunds, big int "siacoinclaimbalance": "9001", // hastings, big int } ``` #### /wallet/033x [POST] loads a v0.3.3.x wallet into the current wallet, harvesting all of the secret keys. All spendable addresses in the loaded wallet will become spendable from the current wallet. ###### Query String Parameters [(with comments)](/doc/api/Wallet.md#query-string-parameters) ``` source encryptionpassword ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /wallet/address [GET] gets a new address from the wallet generated by the primary seed. An error will be returned if the wallet is locked. ###### JSON Response [(with comments)](/doc/api/Wallet.md#json-response-1) ```javascript { "address": "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab" } ``` #### /wallet/addresses [GET] fetches the list of addresses from the wallet. ###### JSON Response [(with comments)](/doc/api/Wallet.md#json-response-2) ```javascript { "addresses": [ "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ] } ``` #### /wallet/backup [GET] creates a backup of the wallet settings file. Though this can easily be done manually, the settings file is often in an unknown or difficult to find location. The /wallet/backup call can spare users the trouble of needing to find their wallet file. ###### Parameters [(with comments)](/doc/api/Wallet.md#query-string-parameters-1) ``` destination ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /wallet/init [POST] initializes the wallet. After the wallet has been initialized once, it does not need to be initialized again, and future calls to /wallet/init will return an error. The encryption password is provided by the api call. If the password is blank, then the password will be set to the same as the seed. ###### Query String Parameters [(with comments)](/doc/api/Wallet.md#query-string-parameters-2) ``` encryptionpassword dictionary // Optional, default is english. ``` ###### JSON Response [(with comments)](/doc/api/Wallet.md#json-response-3) ```javascript { "primaryseed": "hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello" } ``` #### /wallet/seed [POST] gives the wallet a seed to track when looking for incoming transactions. The wallet will be able to spend outputs related to addresses created by the seed. The seed is added as an auxiliary seed, and does not replace the primary seed. Only the primary seed will be used for generating new addresses. ###### Query String Parameters [(with comments)](/doc/api/Wallet.md#query-string-parameters-3) ``` encryptionpassword dictionary seed ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /wallet/seeds [GET] returns the list of seeds in use by the wallet. The primary seed is the only seed that gets used to generate new addresses. This call is unavailable when the wallet is locked. ###### Query String Parameters [(with comments)](/doc/api/Wallet.md#query-string-parameters-4) ``` dictionary ``` ###### JSON Response [(with comments)](/doc/api/Wallet.md#json-response-4) ```javascript { "primaryseed": "hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello", "addressesremaining": 2500, "allseeds": [ "hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello", "foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo", ] } ``` #### /wallet/siacoins [POST] sends siacoins to an address. The outputs are arbitrarily selected from addresses in the wallet. ###### Query String Parameters [(with comments)](/doc/api/Wallet.md#query-string-parameters-5) ``` amount // hastings destination // address ``` ###### JSON Response [(with comments)](/doc/api/Wallet.md#json-response-5) ```javascript { "transactionids": [ "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ] } ``` #### /wallet/siafunds [POST] sends siafunds to an address. The outputs are arbitrarily selected from addresses in the wallet. Any siacoins available in the siafunds being sent (as well as the siacoins available in any siafunds that end up in a refund address) will become available to the wallet as siacoins after 144 confirmations. To access all of the siacoins in the siacoin claim balance, send all of the siafunds to an address in your control (this will give you all the siacoins, while still letting you control the siafunds). ###### Query String Parameters [(with comments)](/doc/api/Wallet.md#query-string-parameters-6) ``` amount // siafunds destination // address ``` ###### JSON Response [(with comments)](/doc/api/Wallet.md#json-response-6) ```javascript { "transactionids": [ "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ] } ``` #### /wallet/siagkey [POST] loads a key into the wallet that was generated by siag. Most siafunds are currently in addresses created by siag. ###### Query String Parameters [(with comments)](/doc/api/Wallet.md#query-string-parameters-7) ``` encryptionpassword keyfiles ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /wallet/lock [POST] locks the wallet, wiping all secret keys. After being locked, the keys are encrypted. Queries for the seed, to send siafunds, and related queries become unavailable. Queries concerning transaction history and balance are still available. ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /wallet/transaction/___:id___ [GET] gets the transaction associated with a specific transaction id. ###### Path Parameters [(with comments)](/doc/api/Wallet.md#path-parameters) ``` :id ``` ###### JSON Response [(with comments)](/doc/api/Wallet.md#json-response-7) ```javascript { "transaction": { "transaction": { // See types.Transaction in https://github.com/NebulousLabs/Sia/blob/master/types/transactions.go }, "transactionid": "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "confirmationheight": 50000, "confirmationtimestamp": 1257894000, "inputs": [ { "fundtype": "siacoin input", "walletaddress": false, "relatedaddress": "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab", "value": "1234", // hastings or siafunds, depending on fundtype, big int } ], "outputs": [ { "fundtype": "siacoin output", "maturityheight": 50000, "walletaddress": false, "relatedaddress": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "value": "1234", // hastings or siafunds, depending on fundtype, big int } ] } } ``` #### /wallet/transactions [GET] returns a list of transactions related to the wallet in chronological order. ###### Query String Parameters [(with comments)](/doc/api/Wallet.md#query-string-parameters-8) ``` startheight // block height endheight // block height ``` ###### JSON Response [(with comments)](/doc/api/Wallet.md#json-response-8) ```javascript { "confirmedtransactions": [ { // See the documentation for '/wallet/transaction/:id' for more information. } ], "unconfirmedtransactions": [ { // See the documentation for '/wallet/transaction/:id' for more information. } ] } ``` #### /wallet/transactions/___:addr___ [GET] returns all of the transactions related to a specific address. ###### Path Parameters [(with comments)](/doc/api/Wallet.md#path-parameters-1) ``` :addr ``` ###### JSON Response [(with comments)](/doc/api/Wallet.md#json-response-9) ```javascript { "transactions": [ { // See the documentation for '/wallet/transaction/:id' for more information. } ] } ``` #### /wallet/unlock [POST] unlocks the wallet. The wallet is capable of knowing whether the correct password was provided. ###### Query String Parameters [(with comments)](/doc/api/Wallet.md#query-string-parameters-9) ``` encryptionpassword ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). Sia-1.0.4/doc/Consensus.md000066400000000000000000000443221302033070500152750ustar00rootroot00000000000000Consensus Rules =============== This document is meant to provide a good high level overview of the Sia cryptosystem, but does not fully explain all of the small details. The most accurate explanation of the consensus rules is the consensus package (and all dependencies). This document will be more understandable if you have a general understanding of proof of work blockchains, and does not try to build up from first principles. Cryptographic Algorithms ------------------------ Sia uses cryptographic hashing and cryptographic signing, each of which has many potentially secure algorithms that can be used. We acknowledge our inexperience, and that we have chosen these algorithms not because of our own confidence in their properties, but because other people seem confident in their properties. For hashing, our primary goal is to use an algorithm that cannot be merge mined with Bitcoin, even partially. A secondary goal is hashing speed on consumer hardware, including phones and other low power devices. For signing, our primary goal is verification speed. A secondary goal is an algorithm that supports HD keys. A tertiary goal is an algorithm that supports threshold signatures. #### Hashing: blake2b [blake2b](http://en.wikipedia.org/wiki/BLAKE_%28hash_function%29#BLAKE2 "Wiki page") has been chosen as a hashing algorithm because it is fast, it has had substantial review, and it has invulnerability to length extension attacks. Another particularly important feature of BLAKE2b is that it is not SHA-2. We wish to avoid merge mining with Bitcoin, because that may result in many apathetic Bitcoin miners mining on our blockchain, which may make soft forks harder to coordinate. #### Signatures: variable type signatures Each public key will have an specifier (a 16 byte array) and a byte slice containing an encoding of the public key. The specifier will tell the signature verification which signing algorithm to use when verifying a signature. Each signature will be a byte slice, the encoding can be determined by looking at the specifier of the corresponding public key. This method allows new signature types to be easily added to the currency in a way that does not invalidate existing outputs and keys. Adding a new signature type requires a hard fork, but allows easy protection against cryptographic breaks, and easy migration to new cryptography if there are any breakthroughs in areas like verification speed, ring signatures, etc. Allowed algorithms: ed25519: The specifier must match the string "ed25519". The public key must be encoded into 32 bytes. Signatures and public keys will need to follow the ed25519 specification. More information can be found at ed25519.cr.yp.to entropy: The specifier must match the string "entropy". The signature will always be invalid. This provides a way to add entropy buffers to SpendCondition objects to protect low entropy information, while being able to prove that the entropy buffers are invalid public keys. There are plans to also add ECDSA secp256k1 and Schnorr secp256k1. New signing algorithms can be added to Sia through a soft fork, because unrecognized algorithm types are always considered to have valid signatures. Currency -------- The Sia cryptosystem has two types of currency. The first is the Siacoin. Siacoins are generated every block and distributed to the miners. These miners can then use the siacoins to fund file contracts, or can send the siacoins to other parties. The siacoin is represented by an infinite precision unsigned integer. The second currency in the Sia cryptosystem is the Siafund, which is a special asset limited to 10,000 indivisible units. Each time a file contract payout is made, 3.9% of the payout is put into the siafund pool. The number of siacoins in the siafund pool must always be divisible by 10,000; the number of coins taken from the payout is rounded down to the nearest 10,000. The siafund is also represented by an infinite precision unsigned integer. Siafund owners can collect the siacoins in the siafund pool. For every 10,000 siacoins added to the siafund pool, a siafund owner can withdraw 1 siacoin. Approx. 8790 siafunds are owned by Nebulous Inc. The remaining siafunds are owned by early backers of the Sia project. There are future plans to enable sidechain compatibility with Sia. This would allow other currencies such as Bitcoin to be spent in all the same places that the Siacoin can be spent. Marshalling ----------- Many of the Sia types need to be hashed at some point, which requires having a consistent algorithm for marshalling types into a set of bytes that can be hashed. The following rules are used for hashing: - Integers are little-endian, and are always encoded as 8 bytes. - Bools are encoded as one byte, where zero is false and one is true. - Variable length types such as strings are prefaced by 8 bytes containing their length. - Arrays and structs are encoded as their individual elements concatenated together. The ordering of the struct is determined by the struct definition. There is only one way to encode each struct. - The Currency type (an infinite precision integer) is encoded in big endian using as many bytes as necessary to represent the underlying number. As it is a variable length type, it is prefixed by 8 bytes containing the length. Block Size ---------- The maximum block size is 2e6 bytes. There is no limit on transaction size, though it must fit inside of the block. Most miners enforce a size limit of 16e3 bytes per transaction. Block Timestamps ---------------- Each block has a minimum allowed timestamp. The minimum timestamp is found by taking the median timestamp of the previous 11 blocks. If there are not 11 previous blocks, the genesis timestamp is used repeatedly. Blocks will be rejected if they are timestamped more than three hours in the future, but can be accepted again once enough time has passed. Block ID -------- The ID of a block is derived using: Hash(Parent Block ID + 64 bit Nonce + Block Merkle Root) The block Merkle root is obtained by creating a Merkle tree whose leaves are the hash of the timestamp, the hashes of the miner outputs (one leaf per miner output), and the hashes of the transactions (one leaf per transaction). Block Target ------------ For a block to be valid, the id of the block must be below a certain target. The target is adjusted once every 500 blocks, and it is adjusted by looking at the timestamps of the previous 1000 blocks. The expected amount of time passed between the most recent block and the 1000th previous block is 10e3 minutes. If more time has passed, the target is lowered. If less time has passed, the target is increased. Each adjustment can adjust the target by up to 2.5x. The target is changed in proportion to the difference in time (If the time was half of what was expected, the new target is 1/2 the old target). There is a clamp on the adjustment. In one block, the target cannot adjust upwards by more more than 1001/1000, and cannot adjust downwards by more than 999/1000. The new target is calculated using (expected time passed in seconds) / (actual time passed in seconds) * (current target). The division and multiplication should be done using infinite precision, and the result should be truncated. If there are not 1000 blocks, the genesis timestamp is used for comparison. The expected time is (10 minutes * block height). Block Subsidy ------------- The coinbase for a block is (300,000 - height) * 10^24, with a minimum of 30,000 \* 10^24. Any miner fees get added to the coinbase to create the block subsidy. The block subsidy is then given to multiple outputs, called the miner payouts. The total value of the miner payouts must equal the block subsidy. The ids of the outputs created by the miner payouts is determined by taking the block id and concatenating the index of the payout that the output corresponds to. The outputs created by the block subsidy cannot be spent for 50 blocks, and are not considered a part of the consensus set until 50 blocks have transpired. This limitation is in place because a simple blockchain reorganization is enough to invalidate the output; double spend attacks and false spend attacks are much easier to execute. Transactions ------------ A Transaction is composed of the following: - Siacoin Inputs - Siacoin Outputs - File Contracts - File Contract Revisions - Storage Proofs - Siafund Inputs - Siafund Outputs - Miner Fees - Arbitrary Data - Transaction Signatures The sum of all the siacoin inputs must equal the sum of all the miner fees, siacoin outputs, and file contract payouts. There can be no leftovers. The sum of all siafund inputs must equal the sum of all siafund outputs. Several objects have unlock hashes. An unlock hash is the Merkle root of the 'unlock conditions' object. The unlock conditions contain a timelock, a number of required signatures, and a set of public keys that can be used during signing. The Merkle root of the unlock condition objects is formed by taking the Merkle root of a tree whose leaves are the timelock, the public keys (one leaf per key), and the number of signatures. This ordering is chosen specifically because the timelock and the number of signatures are low entropy. By using random data as the first and last public key, you can make it safe to reveal any of the public keys without revealing the low entropy items. The unlock conditions cannot be satisfied until enough signatures have provided, and until the height of the blockchain is at least equal to the value of the timelock. The unlock conditions contains a set of public keys which can each be used only once when providing signatures. The same public key can be listed twice, which means that it can be used twice. The number of required signatures indicates how many public keys must be used to validate the input. If required signatures is '0', the input is effectively 'anyone can spend'. If the required signature count is greater than the number of public keys, the input is unspendable. There must be exactly enough signatures. For example, if there are 3 public keys and only two required signatures, then only two signatures can be included into the transaction. Siacoin Inputs -------------- Each input spends an output. The output being spent must exist in the consensus set. The 'value' field of the output indicates how many siacoins must be used in the outputs of the transaction. Valid outputs are miner fees, siacoin outputs, and contract payouts. Siacoin Outputs --------------- Siacoin outputs contain a value and an unlock hash (also called a coin address). The unlock hash is the Merkle root of the spend conditions that must be met to spend the output. File Contracts -------------- A file contract is an agreement by some party to prove they have a file at a given point in time. The contract contains the Merkle root of the data being stored, and the size in bytes of the data being stored. The Merkle root is formed by breaking the file into 64 byte segments and hashing each segment to form the leaves of the Merkle tree. The final segment is not padded out. The storage proof must be submitted between the 'WindowStart' and 'WindowEnd' fields of the contract. There is a 'Payout', which indicates how many siacoins are given out when the storage proof is provided. 3.9% of this payout (rounded down to the nearest 10,000) is put aside for the owners of siafunds. If the storage proof is provided and is valid, the remaining payout is put in an output spendable by the 'valid proof spend hash', and if a valid storage proof is not provided to the blockchain by 'end', the remaining payout is put in an output spendable by the 'missed proof spend hash'. All contracts must have a non-zero payout, 'start' must be before 'end', and 'start' must be greater than the current height of the blockchain. A storage proof is acceptable if it is submitted in the block of height 'end'. File contracts are created with a 'Revision Hash', which is the Merkle root of an unlock conditions object. A 'file contract revision' can be submitted which fulfills the unlock conditions object, resulting in the file contract being replaced by a new file contract, as specified in the revision. File Contract Revisions ----------------------- A file contract revision modifies a contract. File contracts have a revision number, and any revision submitted to the blockchain must have a higher revision number in order to be valid. Any field can be changed except for the payout - siacoins cannot be added to or removed from the file contract during a revision, though the destination upon a successful or unsuccessful storage proof can be changed. The greatest application for file contract revisions is file-diff channels - a file contract can be edited many times off-blockchain as a user uploads new or different content to the host. This improves the overall scalability of Sia. Storage Proofs -------------- A storage proof transaction is any transaction containing a storage proof. Storage proof transactions are not allowed to have siacoin or siafund outputs, and are not allowed to have file contracts. When creating a storage proof, you only prove that you have a single 64 byte segment of the file. The piece that you must prove you have is chosen randomly using the contract id and the id of the 'trigger block'. The trigger block is the block at height 'Start' - 1, where 'Start' is the value 'Start' in the contract that the storage proof is fulfilling. The file is composed of 64 byte segments whose hashes compose the leaves of a Merkle tree. When proving you have the file, you must prove you have one of the leaves. To determine which leaf, take the hash of the contract id concatenated to the trigger block id, then take the numerical value of the result modulus the number of segments: Hash(file contract id + trigger block id) % num segments The proof is formed by providing the 64 byte segment, and then the missing hashes required to fill out the remaining tree. The total size of the proof will be 64 bytes + 32 bytes * log(num segments), and can be verified by anybody who knows the root hash and the file size. Storage proof transactions are not allowed to have siacoin outputs, siafund outputs, or contracts. All outputs created by the storage proofs cannot be spent for 50 blocks. These limits are in place because a simple blockchain reorganization can change the trigger block, which will invalidate the storage proof and therefore the entire transaction. This makes double spend attacks and false spend attacks significantly easier to execute. Siafund Inputs -------------- A siafund input works similar to a siacoin input. It contains the id of a siafund output being spent, and the unlock conditions required to spend the output. A special output is created when a siafund output is used as input. All of the siacoins that have accrued in the siafund since its last spend are sent to the 'claim spend hash' found in the siafund output, which is a normal siacoin address. The value of the siacoin output is determined by taking the size of the siacoin pool when the output was created and comparing it to the current size of the siacoin pool. The equation is: ((Current Pool Size - Previous Pool Size) / 10,000) * siafund quantity Like the miner outputs and the storage proof outputs, the siafund output cannot be spent for 50 blocks because the value of the output can change if the blockchain reorganizes. Reorganizations will not however cause the transaction to be invalidated, so the ban on contracts and outputs does not need to be in place. Siafund Outputs --------------- Like siacoin outputs, siafund outputs contain a value and an unlock hash. The value indicates the number of siafunds that are put into the output, and the unlock hash is the Merkle root of the unlock conditions object which allows the output to be spent. Siafund outputs also contain a claim unlock hash field, which indicates the unlock hash of the siacoin output that is created when the siafund output is spent. The value of the output that gets created will depend on the growth of the siacoin pool between the creation and the spending of the output. This growth is measured by storing a 'claim start', which indicates the size of the siafund pool at the moment the siafund output was created. Miner Fees ---------- A miner fee is a volume of siacoins that get added to the block subsidy. Arbitrary Data -------------- Arbitrary data is a set of data that is ignored by consensus. In the future, it may be used for soft forks, paired with 'anyone can spend' transactions. In the meantime, it is an easy way for third party applications to make use of the siacoin blockchain. Transaction Signatures ---------------------- Each signature points to a single public key index in a single unlock conditions object. No two signatures can point to the same public key index for the same set of unlock conditions. Each signature also contains a timelock, and is not valid until the blockchain has reached a height equal to the timelock height. Signatures also have a 'covered fields' object, which indicates which parts of the transaction get included in the signature. There is a 'whole transaction' flag, which indicates that every part of the transaction except for the signatures gets included, which eliminates any malleability outside of the signatures. The signatures can also be individually included, to enforce that your signature is only valid if certain other signatures are present. If the 'whole transaction' is not set, all fields need to be added manually, and additional parties can add new fields, meaning the transaction will be malleable. This does however allow other parties to add additional inputs, fees, etc. after you have signed the transaction without invalidating your signature. If the whole transaction flag is set, all other elements in the covered fields object must be empty except for the signatures field. The covered fields object contains a slice of indexes for each element of the transaction (siacoin inputs, miner fees, etc.). The slice must be sorted, and there can be no repeated elements. Entirely nonmalleable transactions can be achieved by setting the 'whole transaction' flag and then providing the last signature, including every other signature in your signature. Because no frivolous signatures are allowed, the transaction cannot be changed without your signature being invalidated. Sia-1.0.4/doc/Developers.md000066400000000000000000000230301302033070500154160ustar00rootroot00000000000000Developer Environment ===================== Sia is written in Go. To build and test Sia, you are going to need a working go environment, including having both $GOROOT/bin and $GOPATH/bin in your $PATH. For most Linux distributions, Go will be in the package manager, though it may be an old version that is incompatible with Sia. Once you have a working Go environment, you are set to build the project. If you plan on cross compiling Sia, you may need to install Go from source. You can find information on that [here](http://golang.org/doc/install/source). Sia has has a development build, an automated testing build, and a release build. The release build is the only one that can synchronize to the full network. To get the release build, it is usually sufficient to run `go get -u github.com/NebulousLabs/Sia/...`. This will download Sia and its dependencies and install binaries in $GOPATH/bin. After downloading, you can find the Sia source code in $GOPATH/src/github.com/NebulousLabs/Sia. To build the release binary, run `make release-std` from this directory. To build the release binary with a (slow) race detector and an array of debugging asserts, run `make release`. To build the developer binary (which has a different genesis block, faster block times, and a few other tweaks), just run `make`. If you intend to contribute to Sia, you should start by forking the project on GitHub, and then adding your fork as a "remote" in the Sia git repository via `git remote add [fork name] [fork url]`. Now you can develop by pulling changes from `origin`, pushing your modifications to `[fork name]`, and then making a pull request on GitHub. If you see an error like the one below, it means that you either forgot to run `make dependencies`, or you cloned the project into a path that the go tool does not recognize (usually the wrong path, or symbolic links were somehow involved). ``` consensus/fork.go:4:2: cannot find package "github.com/NebulousLabs/Sia/crypto" in any of: /usr/lib/go/src/github.com/NebulousLabs/Sia/crypto (from $GOROOT) /home/user/gopath/src/github.com/NebulousLabs/Sia/crypto (from $GOPATH) ``` Developer Conventions ===================== This file is meant to help a developer navigate the codebase and develop clean, maintainable code. Knowing all of these conventions will also make it easier to read and code review the Sia project. The primary purpose of the conventions within Sia is to keep the codebase simple. Simpler constructions means easier code reviews, greater accessibility to newcomers, and less potential for mistakes. It is also to keep things uniform, much in the spirit of 'go fmt'. When everything looks the same, everyone has an easier time reading and reviewing code they did not write themselves. Documentation ------------- All structs, functions, and interfaces must have a docstring. Anytime that something is left unfinished, place a comment containing the string 'TODO:'. This sends a clear message to other developers, and creates a greppable way to find unfinished parts of the codebase. 'TODO' statements are currently discouraged. As the codebase matures, 'TODO' statements will become increasingly frowned upon. 'TODO' statements should not document feature requests, but instead document incompleteness where the incompleteness causes disruption to user experience or causes a security vulnerability. Documentation should give a sense of what each function does, but should also give a sense of the overall architecture of the code. Where useful, examples should be provided, and common pitfalls should be explained. Anything that breaks other conventions in any way needs to have a comment, even if it is obvious why the convention had to be broken. The goal of the codebase is to be accessible to newbies. Anything more advanced than what you would expect to remember from an 'Intro to Data Structures' class should have an explanation about what the concept it is and why it was picked over other potential choices. Code that exists purely to be compatible with previous versions of the software should be tagged with a 'COMPATvX.X.X' comment. Examples below. ```go // Find and sort the outputs. outputs := getOutputs() // TODO: actually sort the outputs. ``` ```go // Disallow unknown agents. // // COMPATv0.4.0: allow a blank agent to preserve compatibility with // 'siac' v0.4.0, which did not set an agent. if agent != "SiaAgent" && agent != "" { return errors.New("unrecognized agent!") } ``` Naming ------ Names are used to give readers and reviewers a sense of what is happening in the code. When naming variables, you should assume that the person reading your code is unfamiliar with the codebase. Short names (like 'cs' instead of 'consensusSet') should only be used when the context is immediately obvious. For example 'cs := new(ConsensusSet)' is immediately obvious context for 'cs', and so 'cs' is appropriate for the rest of the function. Data structures should never have shortened names. 'FileContract.mr' is confusing to anyone who has not used the data structure extensively. The code should be accessible to people who are unfamiliar with the codebase. One exception is for the variable called 'mu', which is short for 'mutex'. This exception is made because 'mu' appears in many data structures. When calling functions with obscure parameters, named variables should be used to indicate what the parameters do. For example, 'm := NewMiner(1)' is confusing. Instead, use 'threads := 1; m := NewMiner(threads)'. The name gives readers a sense of what the parameter within 'NewMiner' does even when they are not familiar with the 'NewMiner' function. Where possible, functions with obscure, untyped inputs should be avoided. The most important thing to remember when choosing names is to cater to people who are unfamiliar with the code. A reader should never have to ask 'What is `cs`?' on their first pass through the code, even though to experienced developers it is obvious that `cs` refers to a consensus.ConsensusSet. Control Flow ------------ Where possible, control structures should be minimized or avoided. This includes avoiding nested if statements, and avoiding else statements where possible. Sometimes, complex control structures are necessary, but where possible use alternative code patterns and insert functions to break things up. Example: ```go // Do not do this: if err != nil { return } else { forkBlockchain(node) } // Instead to this: if err != nil { return } forkBlockchain(node) ``` Mutexes ------- All exported functions from a package and/or object need to be thread safe. Usually, this means that the first lines of the function contain a `Lock(); defer Unlock()`. Simple locking schemes should be preferred over performant locking schemes. As will everything else, anything unusual or convention breaking should have a comment. Non-exported functions should not do any locking, unless they have a special prefix to the name (explained below). The responsibility for thread-safety comes from the exported functions which call the non-exported functions. Maintaining this convention minimizes developer overhead when working with complex objects. Functions prefixed 'threaded' (example 'threadedMine') are meant to be called in their own goroutine (`go threadedMine()`) and will manage their own thread-safety. Error Handling -------------- All errors need to be checked as soon as they are received, even if they are known to not cause problems. The statement that checks the error needs to be `if err != nil`, and if there is a good reason to use an alternative statement (such as `err == nil`), it must be documented. The body of the if statement should be at most 4 lines, but usually only one. Anything requiring more lines needs to be its own function. Example: ```go block, err := s.AcceptBlock() if err != nil { handleAcceptBlockErr(block, err) return } ``` Sanity Checks ------------- Some functions make assumptions. For example, the 'addTransaction' function assumes that the transaction being added is not in conflict with any other transactions. Where possible, these explicit assumptions should be validated. Example: ```go if build.DEBUG { _, exists := tp.usedOutputs[input.OutputID] if exists { panic("incorrect use of addTransaction") } } ``` In the example, a panic is called for incorrect use of the function, but only in debug mode. This failure will be invisible in production code, but the code will have higher performance because the code should never fail anyway. If the code is continually checking items that should be universally true, mistakes are easier to catch during testing, and side effects are less likely to go unnoticed. Sanity checks and panics are purely to check for developer mistakes. A user should not be able to trigger a panic, and no set of network communications or real-world conditions should be able to trigger a panic. Testing ------- The test suite code should be the same quality as the rest of the codebase. When writing new code in a pull request, the pull request should include test coverage for the code. Most modules have a tester object, which can be created by calling `createXXXTester`. Module testers typically have a consensus set, a miner, a wallet, and a few other relevant modules that can be used to build transactions, mine blocks, etc. In general, testing that uses exclusively exported functions to achieve full coverage is preferred. These types of tests seem to find more bugs and trigger more asserts. Any testing provided by a third party which is both maintainable and reasonably quick will be accepted. There is little downside to more testing, even when the testing is largely redundant. Sia-1.0.4/doc/Encoding.md000066400000000000000000000056171302033070500150470ustar00rootroot00000000000000Encoding ======== The encoding package converts arbitrary objects into byte slices, and vice versa. Objects are encoded as binary data, without type information. The decoder will attempt to decode its input bytes into whatever type it is passed. For example: ```go Marshal(int64(3)) == []byte{3, 0, 0, 0, 0, 0, 0, 0} var x int64 Unmarshal([]byte{3, 0, 0, 0, 0, 0, 0, 0}, &x) // x == 3 ``` Note that this leads to some ambiguity. Since an `int64` and a `uint64` are both 8 bytes long, it is possible to encode an `int64` and successfully decode it as a `uint64`. As a result, it is imperative that *the decoder knows exactly what it is decoding*. Developers must rely on context to determine what type to decode into. The specific rules for encoding Go's builtin types are as follows: Integers are little-endian, and are always encoded as 8 bytes, i.e. their `int64` or `uint64` equivalent. Booleans are encoded as one byte, either zero (false) or one (true). No other values may be used. Nil pointers are equivalent to "false," i.e. a single zero byte. Valid pointers are represented by a "true" byte (0x01) followed by the encoding of the dereferenced value. Variable-length types, such as strings and slices, are represented by an 8-byte unsigned length prefix followed by the encoded value. Strings are encoded as their literal UTF-8 bytes. Slices are encoded as the concatenation of their encoded elements. For example: ```go // slice len: 1 string len: 3 string data Marshal([]string{"foo"}) == []byte{1,0,0,0,0,0,0,0, 3,0,0,0,0,0,0,0, 'f','o','o'} ``` Maps are not supported; attempting to encode a map will cause `Marshal` to panic. This is because their elements are not ordered in a consistent way, and it is imperative that this encoding scheme be deterministic. To encode a map, either convert it to a slice of structs, or define a `MarshalSia` method (see below). Arrays and structs are simply the concatenation of their encoded elements. Byte slices are not subject to the 8-byte integer rule; they are encoded as their literal representation, one byte per byte. All struct fields must be exported. (For some types this is a bit awkward, so this rule is subject to change.) The ordering of struct fields is determined by their type definition. For example: ```go type foo struct { S string I int } Marshal(foo{"bar", 3}) == append(Marshal("bar"), Marshal(3)...) ``` Finally, if a type implements the SiaMarshaler interface, its MarshalSia method will be used to encode the type. Similarly, if a type implements the SiaUnmarshal interface, its UnmarshalSia method will be used to decode the type. Note that unless a type implements both interfaces, it must conform to the spec above. Otherwise, it may encode and decode itself however desired. This may be an attractive option where speed is critical, since it allows for more compact representations, and bypasses the use of reflection. Sia-1.0.4/doc/File Contract Negotiation.md000066400000000000000000000343641302033070500202000ustar00rootroot00000000000000File Contract Negotiation ========================= Securing data on Sia requires creating and revising file contracts in an untrusted environment. Managing data on Sia happens through several protocols: + Settings Request - the host sends the renter its settings. + Revision Request - the renter will send the host a file contract id, and the host will send the most recent file contract revision that it knows of for that file contract, with the signatures. A challenge and response is also performed to verify that the renter is able to create the signatures to modify the file contract revision. + File Contract Creation - no data is uploaded during the initial creation of a file contract, but funds are allocated so that the file contract can be iteratively revised in the future. + File Contract Revision - an existing file contract is revised so that data can be added to an arbitrary place, or removed from an arbitrary place. + File Contract Renewal - an existing file contract is renewed, meaning that a new file contract with a different id is created, but that has the same data. New funds are added to this file contract, and it can now be modified separately from the previous contract. + Data Request - data is requested from the host by hash. + (planned for later) Storage Proof Request - the renter requests that the host perform an out-of-band storage proof. + (planned for later) Metadata Request - the renter requests some metadata about the file contract from the host, namely the list of hashes that compose the file. This list of hashes is provided along with a cryptographic proof that the hashes are valid. The proof is only needed if only a subset of hashes are being sent. A frequently seen construction is 'acceptance'. The renter or host may have the opportunity to accept or reject a communication, which takes the form of a string. The acceptance string is always the same, and any string that is not the acceptance string is a rejection. The rejection string can include reasons why the rejection occurred, but most not exceed 255 bytes. After a rejection, the connection is always closed. The protocols described below are numbered. The number indicates when the communicator is switching. Each pair of numbers is a full round trip of communications. All communications attempt to support slow connections and Tor connections. Any connection with a throughput below 100kbps may struggle to perform the uploads and downloads, and any connection with a rountrip latency greater than 2 minutes may struggle to complete the protocols. Settings Request ---------------- The host signs the settings request to prove that the connection has opened to the right party. Hosts announce on the blockchain and perform burn, therefore identity is important. 1. The renter makes an RPC to the host, opening a connection. The connection deadline should be at least 120 seconds. 2. The host sends the renter the most recent copy of its external settings, signed by the host public key. The connection is then closed. Revision Request ---------------- The renter requests a recent revision from the host. Often, this request precedes modifications. A file contract can only be open for revision with one party at a time. To prevent DoS attacks, the party must authenticate here by performing a challenge-response protocol during the revision request. Putting this challenge-response requirement in the revision-request can help improve privacy, though the host is under no cryptographic or incentive-based obligation to preserve the privacy of the revision. 1. The renter makes an RPC to the host, opening a connection. The connection deadline should be at least 120 seconds. The renter sends the file contract id for the revision being requested. 2. The host writes 32 bytes of random data that the renter must sign for the renter key in the corresponding file contract. 3. The renter returns the signed challenge. 4. The host verifies the signature from the renter and then sends the renter the most recent file contract revision, along with the transaction signatures from both the renter and the host. The connection is then closed. File Contract Creation ---------------------- A few decisions were made regarding the file contract protocol. The first is that the renter should not sign the file contract until the host has formally accepted the file contract. The second is that the host should be the last one to sign the file contract, as the renter is the party with the strong reputation system. Instead of sending a whole transaction each time, the transaction is sent piecemeal, and only the new parts at each step are sent to the other party. This minimizes the surface area of data for a malicious party to manipulate, which means less verification code, which means less chances of having a bug in the verification code. The renter pays for the siafund fee on the host's collateral and contract fee. If a renter opens a file contract and then never uses it, the host does not lose money. This does put the renter at risk, as they may open up a file contract and then watch the host leave, but the renter is spreading the risk over communications with many hosts, and has a reputation system that will help ensure that the renter is only dealing with upstanding hosts. 1. The renter makes an RPC to the host, opening a connection. The connection deadline should be at least 360 seconds. 2. The host sends the renter the most recent copy of its settings, signed. If the host is not accepting new file contracts, the connection is closed. 3. The renter sends a notice of acceptance or rejection. If the renter accepts, the renter then sends a funded file contract transaction without a signature, followed by the public key that will be used to create the renter's portion of the UnlockConditions for the file contract. 4. The host sends an acceptance or rejection of the file contract. If the host accepts, the host will add collateral to the file contract, and will send the renter the inputs + outputs for the collateral, followed by any new parent transactions. The length of any of these may be zero. 5. The renter indicates acceptance or rejection of the file contract. If the renter accepts, the renter will sign the file contract and send the transaction signatures to the host. The renter will also send a signature for a no-op file contract revision that follows the file contract. 6. The host may only reject the file contract in the event that the renter has sent invalid signatures, so the acceptance step is skipped. The host signs the file contract and sends the transaction signatures to the renter, and the host creates and sends a signature for the no-op revision that follows the file contract. The connection is closed. File Contract Revision ---------------------- 1. The renter makes an RPC to the host, opening a connection. The minimum deadline for the connection is 600 seconds. The renter then sends a file contract ID, indicating the file contract that is getting revised during the RPC. 2. The host will respond with a 32 byte challenge - a random 32 bytes that the renter will need to sign. 3. The renter will sign the challenge with the renter key that protects the file contract. This is to prove that the renter has access to the file contract. 4. The host will verify the challenge signature, then send an acceptance or rejection. If accetped, the host will send the most recent file contract revision for the file contract along with the transaction signagtures that validate the revision. The host will lock the file contract, meaning no other changes can be made to the revision file contract until this connection has closed. A loop begins. The host sends the most recent revision of the host settings to the renter, signed. The settings are sent after each iteration of the loop to enable high resolution dynamic pricing for the host, especially for bandwidth. 6. The renter may reject or accept the settings + revision. A specific rejection message will gracefully terminate the loop here. The renter will send an unsigned file contract revision followed by a batch of modification actions which the revision pays for. Batching allows the renter to send a lot of data in a single, one-way connection, improving throughput. The renter will send a number indicating how many modifications will be made in a batch, and then sends each modification in order. A single modification can either be an insert, a modify, or a delete. An insert is an index, indicating the index where the data is going to be inserted. '0' indicates that the data is inserted at the very beginning, '1' indicates that the data will be inserted between the first and second existing sectors, etc. The index is followed by the 4MB of data. A modify is an index indicating which sector is being modified, followed by an offset indicating which data within the sector is being modified. Finally, some data is provided indicating what the data in the sector should be replaced with starting from that offset. The offset + len of the data should not exceed the sector size of 4MB. A delete is an index indicating the index of the sector that is being deleted. Each operation within a batch is atomic, meaning if you are inserting 3 sectors at the front of the file contract, the indexes of each should be '0', '1', '2'. 7. The host indicates either acceptance or rejection of the new revision. 8. The renter signs the revision and sends the signature to the host. 9. The host signs the revision and sends the siganture to the renter. Both parties submit the new revision to the transaction pool. The connection deadline is reset to 600 seconds (unless the maximum deadline has been reached), and the loop restarts. File Contract Renewal --------------------- 1. The renter makes an RPC to the host, opening a connection. The minimum deadline for the connection is 600 seconds. The renter then sends a file contract ID, indicating the file contract that is getting revised during the RPC. 2. The host will respond with a 32 byte challenge - a random 32 bytes that the renter will need to sign. 3. The renter will sign the challenge with the renter key that protects the file contract. This is to prove that the renter has access to the file contract. 4. The host will verify the challenge signature, then send an acceptance or rejection. If accetped, the host will send the most recent file contract revision for the file contract along with the transaction signagtures that validate the revision. The host will lock the file contract, meaning no other changes can be made to the revision file contract until this connection has closed. The host sends the most recent revision of the host settings to the renter, signed. If the host is not accepting new file contracts, the connection is closed. 5. The renter either accepts or rejects the settings. If accepted, the renter sends a funded, unsigned file contract to the host, containing the same Merkle root as the previous file contract, and also containing a renewed payout with conditional payments to the host to cover the host storing the data for the extended duration. 6. The host will accept or reject the renewed file contract. If accepted, the host will add collateral (and miner fees if desired) and send the inputs + outputs for the collateral, along with any new parent transactions. The length of any of these may be zero. 7. The renter will accept or reject the host's additions. If accepting, the renter will send signatures for the transaction to the host. The renter will also send a signature for a no-op file contract revision that follows the file contract. 8. The host may only reject the file contract in the event that the renter has sent invalid signatures, so the acceptance step is skipped. The host signs the file contract and sends the transaction signatures to the renter, and the host creates and sends a signature for the no-op revision that follows the file contract. The connection is closed. Data Request ------------ 1. The renter makes an RPC to the host, opening a connection. The connection deadline is at least 600 seconds. The renter will send a file contract id corresponding to the file contract that will be used to pay for the download. 2. The host will respond with a 32 byte challenge - a random 32 bytes that the renter will need to sign. 3. The renter will sign the challenge with the public key that protects the file contract being used to pay for the download. This proves that the renter has access to the payment. 4. The host will verify the challenge signature, and then send an acceptance or rejection. If accepted, the host will send the most recent file contract revision followed by the signautres that validate the revision. The host will lock the file contract, preventing other connections from making changes to the underlying storage obligation. A loop begins. The host sends the most recent external settings to the renter, signed. The settings are sent each iteration to provide high resolution dynamic bandwidth pricing. 5. The host will send the renter the most recent file contract revision, along with the signatures that validate the revision. A loop begins, which will allow the renter to download multiple batches of data from the same connection. The host will send the host settings, and the most recent file contract revision transaction. If there is no revision yet, the host will send a blank transaction. The host is expected to always have the most recent revision (the host signs last), the renter may not have the most recent revision. 6. The renter will accept or reject the host's settings. If accepting, the renter will send a file contract revision, unsigned, to pay for the download request. The renter will then send the download request itself. 7. The host will either accept or reject the revision. 8. The renter will send a signature for the file contract revision. 9. The host sends a signature for the file contract revision, followed by the data that was requested by the download request. The loop starts over, and the connection deadline is reset to a minimum of 600 seconds. Sia-1.0.4/doc/Guide to Contributing to Sia.md000066400000000000000000000217441302033070500205100ustar00rootroot00000000000000# Contributing to Sia #### Table of Contents * [Get started with Go](#go) * [Install Go](#install-go) * [Learn Go]("learn-go") * [Build Sia](#build-sia) * [Contribute to the codebase](#contribute) * [Set up git](#setup-git) * [Fork the Sia repository](#fork) * [Write some code](#write) * [Submit your code for review](#pull) * [More git resources](#git-resources) * [Where to start](#where-to-start) * [Contact us](#contact) ## Get started with Go ### Install Go To install Go on your computer, follow the [official installation guide][install-go]. You should install the latest [official Go binary][binary] for your system (if not available, [install from source][source]). If you plan to cross compile Sia, see [Cross Compilation with Go 1.5][cross] by Dave Cheney. Now make a workspace directory in which you will store source code and dependencies. You can choose any filepath except where you installed Go (don't choose `/usr/local`). ```bash # make a working directory called golang in your home directory $ mkdir $HOME/golang # store base path in an environmental variable $ echo 'export GOPATH=$HOME/golang' >> $HOME/.profile # add bin subdirectory to PATH environmental variable $ echo 'export PATH=$PATH:$GOPATH/bin' >> $HOME/.profile ``` ### Learn Go * To get familiar with the language, start with the official [Tour of Go][tour]. * Move onto [How to Write Go Code][how] to learn how to organize Go packages and use the go tool. * Finish with the [Effective Go][effective] guide. ## Build Sia To build Sia on your machine, enter the following on the command line: ```bash # Download Sia and its dependencies # Binaries will be installed in $GOPATH/bin $ go get -u github.com/NebulousLabs/Sia/... # Switch to directory containing Sia source code $ cd $GOPATH/src/github.com/NebulousLabs/Sia # You have three Sia builds to choose from. # To build the standard release binary: $ make release-std # Or to build the release binary with race detection and an array debugging # asserts: $ make release # Or to build the developer binary (with a different genesis block, faster # block times, and other changes): $ make ``` ## Contribute to the codebase ### Set up git Install git on your machine according to [these instructions][install-git] in the Pro Git book. You will first need to set up global settings using the command line. ```bash $ git config --global user.name "Your Name" $ git config --global user.email you@somedomain.com # Tell git to remember your login information for a certain amount of time. # Default time is 15 minutes: $ git config --global credential.helper cache # Or you can choose a different amount of time: $ git config --global credential.helper "cache --timeout=[seconds]" ``` ### Fork the Sia repository While logged into your Github account, navigate to the [Sia repository][sia] and click the 'Fork' button in the upper righthand corner. Your account now has a 'forked' copy of the original repo at `https://github.com//Sia`. When you installed Sia using `go get`, the go tool put the Sia source code in $GOPATH/src/github.com/NebulousLabs/Sia. Change to that directory and set up your fork as a git [remote][remote]: ```bash $ cd $GOPATH/src/github.com/NebulousLabs/Sia # Add your fork as a remote. Name it whatever is convenient, # e.g your GitHub username $ git remote add https://github.com//Sia.git ``` ### Write some code Right now your git local repository only has one branch (called 'master' by default). If you want to make changes, add a new branch and make your changes there. You should maintain master as an up-to-date copy of the NebulousLabs/Sia repository's master branch. To create and checkout a new branch: ```bash # If you're not already in the right directory: $ cd $GOPATH/src/NebulousLabs/Sia # Make sure you're on branch master $ git checkout master # Create and checkout a new branch $ git checkout -b ``` Now write some code while the new branch is checked out. Only implement one logical change per branch. If you're working on several things at once, make multiple branches. To switch between branches you're working on, you have to stash the changes in the branch you're switching from by running `git stash`, which tucks away all changes since the last commit. ```bash # Stash changes to current branch. $ git stash # Checkout other branch. $ git checkout ... # Make changes ... # Return to first branch: $ git checkout # View a list of stashes and their corresponding hashes. $ git stash list # Reapply changes from the stash you want to recover and remove that stash from. # the list $ git stash pop ``` To learn more about branching, see [Using the Fork-and-Branch Git Workflow][branch] and [Pro Git - Branches in a Nutshell][nutshell]. For more on stashing, see [Pro Git - Stashing and Cleaning][stashing]. Be sure to follow the conventions detailed in [docs/Developers.md][developers.md]. We will reject pull requests that do not satisfy these best practices. Once you've finished making changes, stage and commit your changes then update your fork on Github: ```bash # Make sure the code is up to date with the original repo: $ git checkout master $ git pull origin master # Checkout branch with changes. $ git checkout $ git rebase master # Before every pull request, you should run `make test-long` # to test your code and fix formatting and style problems. $ make test-long # If all goes well, proceed to staging your changed files: $ git add # Use `git status` to see what files have been staged. $ git status # Commit your changes. If you just run `commit`, a text editor will pop up for # you to enter a description of your changes. $ git commit -m "Add new tests for CommitSync method" # Push the changes to your fork on GitHub, which you should have set up as a # remote already. $ git push ``` ### Submit your code for review Once you've tested your new code and pushed changes to your fork, navigate to your fork at `https://github.com//Sia` in your browser. Switch to the branch you've made changes on by selecting it from the list on the upper left. Then click 'New pull request' on the upper right. Once you have made the pull request, we will review your code. We will reject code that is unsafe, difficult to read, or otherwise violates the conventions outlined in [docs/Developers.md][developers.md]. Here's a sample code review comment: ![Screenshot](assets/codereview.png) If you want to tweak code for which you've already submitted a pull request, push the updated code to your fork with `git push -f ` and summarize the changes you've made in a comment on the pull request page on GitHub. Once we have accepted your changes and merged them into the original repo, you have some cleanup to do: ```bash # Update local master branch to reflect changes in origin (the original # repo). $ git pull origin master # Delete the branch you made the pull request from. $ git branch -d # Delete the remote branch on your fork. $ git push : # Update your fork. $ git push master ``` ### More Git resources * [How to into git (and Github)][luke] by Luke Champine * [Official resources for learning Git][git] ## Where to start If you'd like to contribute to Sia but don't have any specific ideas, writing tests is a good way to get your feet wet. See [doc/Running and Writing Tests for Sia.md](Running\ and\ Writing\ Tests\ for\ Sia.md) to get started. ## Contact us Feel free to ask for help on the #dev channel on [Slack][slack]. [cross]: http://dave.cheney.net/2015/08/22/cross-compilation-with-go-1-5 [binary]: https://golang.org/dl/ [source]: https://golang.org/doc/install/source [tour]: https://tour.golang.org/welcome/1 [how]: https://golang.org/doc/code.html [luke]: https://gist.github.com/lukechampine/6418449 [git]: https://git-scm.com/doc [cheney]: http://dave.cheney.net/2013/06/09/writing-table-driven-tests-in-go [install-go]: https://golang.org/doc/install [signup]: https://github.com/join?source=header-home [effective]: https://golang.org/doc/effective_go.html [sia]: https://github.com/NebulousLabs/Sia [branch]: http://blog.scottlowe.org/2015/01/27/using-fork-branch-git-workflow/ [developers.md]: https://github.com/NebulousLabs/Sia/blob/master/doc/Developers.md [gofmt]: https://golang.org/cmd/gofmt/ [nutshell]: https://git-scm.com/book/en/v2/Git-Branching-Branches-in-a-Nutshell [slack]: http://slackin.siacoin.com [install-git]: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git [test-doc]: https://github.com/NebulousLabs/Sia/blob/master/doc/Testing.md [stashing]: https://git-scm.com/book/en/v2/Git-Tools-Stashing-and-Cleaning [remote]: https://git-scm.com/book/en/v2/Git-Basics-Working-with-Remotes Sia-1.0.4/doc/Modules.md000066400000000000000000000037301302033070500147230ustar00rootroot00000000000000Module Conventions ================== Each module has a file/directory where they store persistent data (if necessary). When module.New is called, the module is responsible for creating and populating that directory. The logic for saving and loading data belongs in persist.go. Modules that depend on external information (such as the state of consensus) have an update.go to manage fetching and integrating the external information. If that information is coming from another module, a subscription should be used. Module subscription uses a ModuleSubscriber interface (which the subscriber must satisfy) and a ModuleSubscribe method (implemented by the parent module). As the parent module gets updates, it will call ReceiveModuleUpdate (the only method of the ModuleSubscriber interface) on all subscribers, taking care that each subscriber always receives the updates in the correct order. This method of subscription is chosen to keep information flow simple and synchronized - a child module should never have information that the parent module does not (it just causes problems). For testing, it is often important to know that an update has propagated to all modules. Any module that subscribes to another must also implement a ModuleNotify function in subscriptions.go. ModuleNotify returns a channel down which a struct{} will be sent every time that module receives an update from a parent module. To keep things simple, a module should not subscribe to the parent of another module that it is subscribed to. For example, the transaction pool is subscribed to the consensus set. Therefore, no module should subscribe to both the transaction pool and the consensus set. All consensus set updates should be received through the transaction pool. This helps with synchronization and ensures that no child module ever has information that the parent module has not yet received (desynchronization). #### Module Update Flow consensus -> (host, hostdb, renter, (transaction pool -> miner, wallet)) Sia-1.0.4/doc/Running and Writing Tests for Sia.md000066400000000000000000000227701302033070500214160ustar00rootroot00000000000000# Running and Writing Tests for Sia Improving test coverage is a great way to start contributing to Sia. This guide focuses on how to write tests. To learn about making pull requests to submit the code you've written, see [doc/Guide to Contributing to Sia.md][guide]. You should also read [doc/Developers.md][developers] to learn about Sia code conventions and quality standards. #### Table of Contents * [Running tests for Sia](#existing) * [Updating code before testing](#update) * [Testing the entire build](#entire) * [Testing a particular package](#particular) * [Writing new tests for Sia](#write) * [A few guidelines](#naming) * [Basic test format](#basic) * [Table-driven tests](#table) * [Questions?](#questions) ## Running tests for Sia Go's comprehensive [test package][pkg/testing] makes testing straightforward, particularly when you use the bundled tools included in the [Sia makefile][makefile], including `make test`, `make cover`, `make bench`, and their variants. ### Updating code before testing If you just want to run existing tests on the codebase as is, you just need to pull the latest version of the original repo to your master branch. (If that sentence didn't make sense, go read [doc/Guide to Contributing to Sia.md][guide].) ```bash # Make sure you are in the right directory. $ cd $GOPATH/src/github.com//Sia # Also make sure you're working with the right branch. $ git checkout master # Pull latest changes from origin, the original Sia repo. $ git pull origin master # Update your fork of the repo, which should be set up as a remote. $ git push master ``` If you want to run tests on the new code you've added, first make sure the rest of the code is up to date. New code should be on its own branch (again, see [doc/Guide to Contributing to Sia.md][guide]). ```bash # Make sure you are in the right directory. $ cd $GOPATH/src/github.com//Sia # Checkout the branch you made the changes on. $ git checkout # Stash any tracked but uncommitted changes. $ git stash # Then switch back to `master` and update it to match the original repo. $ git checkout master $ git pull origin master # Update your fork of the repo, which you should have set up as a remote. $ git push master # Make the updated `master` the new base of the branch you made the changes on, # which involves reapplying all the commits made to that branch. Without the # `--ignore-date` flag, git rebase changes the date on all the commits to the # current date. $ git checkout $ git rebase master --ignore-date # Restore the changes you stashed earlier. $ git stash pop ``` When you call `rebase`, you may run into some merge conflicts. Luke Champine's ['How to into git and GitHub'][luke] has more details (and many useful tricks). Once the branch you want to test is up to date, you're ready to run some tests. ### Testing the entire build The `make test` command runs all tests (functions starting with `Test` in `_test.go` files) for each package, setting off a panic for any test that runs longer than 5s. For verbose output, run `make test-v` (which panics after 15s instead of 5s). Finally, `make test-long` has verbose output, only panics when a test takes 5 minutes, and also cleans up your code using `gofmt` and `golint`. **You should run** `make test-long` **before each pull request.** Run `make cover` to run all tests for each package and generate color-coded .html visualizations of test coverage by function for each source file. Open `cover/.html` in a browser to inspect a module's test coverage. For example, here's part of the html file generated for the persist package: ![Screenshot](assets/covertool.png) Meanwhile, `make bench` will call `gofmt` on all packages, then run all benchmarks (functions starting with `Benchmark` in `_test.go` files). ### Testing a particular package or function To run tests for just a certain package, run `make test pkgs=./`. To run a certain test function, run `make test pkgs=./ run=`. The same goes for `make test-long`, `make cover` and `make bench`. For example, running `test-long` on the package persist produces this output: ```bash $ make test-long pkgs=./persist rm -rf release doc/whitepaper.aux doc/whitepaper.log doc/whitepaper.pdf gofmt -s -l -w ./persist go install ./persist go vet ./persist go test -v -race -tags='testing debug' -timeout=300s ./persist -run=Test === RUN TestOpenDatabase --- PASS: TestOpenDatabase (0.42s) === RUN TestSaveLoad --- PASS: TestSaveLoad (0.00s) === RUN TestSaveLoadFile --- PASS: TestSaveLoadFile (0.01s) === RUN TestSaveLoadFileSync --- PASS: TestSaveLoadFileSync (0.00s) === RUN TestLogger --- PASS: TestLogger (0.00s) === RUN TestLoggerCritical --- PASS: TestLoggerCritical (0.00s) === RUN TestIntegrationRandomSuffix --- PASS: TestIntegrationRandomSuffix (0.01s) === RUN TestAbsolutePathSafeFile --- PASS: TestAbsolutePathSafeFile (0.00s) === RUN TestRelativePathSafeFile --- PASS: TestRelativePathSafeFile (0.00s) PASS ok github.com/NebulousLabs/Sia/persist 1.485s $ ``` ## Writing new tests for Sia When you run `make cover`, you'll notice that many files have pretty low coverage. We're working on fixing that, but we could use your help. ### A few guidelines * The test functions for `filename.go` should go in `filename_test.go` in the same directory and package. * A test function name should start with `Test` and clearly convey what is being tested. * You should declare function-specific variables and constants locally (inside the test function) instead of globally (outside the test function). [That holds in general][global], not just for tests. * As always, code should adhere to the standards and conventions laid out in [doc/Developers.md][developers]. ### Basic test format Suppose we'd like to test the Bar method belonging to type Foo. ```go // TestFoo checks that the Bar method on type Foo responds correctly to a normal // input and returns the expected error when given a bad input. func TestFoo(t *testing.T) { foo, err := NewFoo() if err != nil { // If NewFoo failed, we can't continue testing. t.Fatal(err) } // Try a normal input; should succeed. err := foo.Bar(3) if err != nil { // Report the error, but don't abort the test. t.Error(err) } // Try a bad input; should return an error. // NOTE: Always prefer to compare to a specific error, rather than // err == nil err = Foo.Bar(0) if err != errDivideByZero { t.Errorf("expected errDivideByZero, got %v", err) } } ``` ### Table-driven tests in Go If you're looking to test a bunch of inputs, write a [table-driven test][table] with a slice of anonymous structs. For example, see `TestParseFileSize` in [siac/parse_test.go][parse_test]: ```go func TestParseFilesize(t *testing.T) { // Define a table of test cases in the form of a slice of anonymous structs. tests := []struct { in, out string err error }{ {"1b", "1", nil}, {"1KB", "1000", nil}, {"1MB", "1000000", nil}, {"1GB", "1000000000", nil}, {"1TB", "1000000000000", nil}, {"1KiB", "1024", nil}, {"1MiB", "1048576", nil}, {"1GiB", "1073741824", nil}, {"1TiB", "1099511627776", nil}, {"", "", errUnableToParseSize}, {"123", "123", nil}, {"123TB", "123000000000000", nil}, {"123GiB", "132070244352", nil}, {"123BiB", "", errUnableToParseSize}, {"GB", "", errUnableToParseSize}, {"123G", "", errUnableToParseSize}, {"123B99", "", errUnableToParseSize}, {"12A3456", "", errUnableToParseSize}, {"1.23KB", "1230", nil}, {"1.234KB", "1234", nil}, {"1.2345KB", "1234", nil}, } // Loop through the table of test cases to make sure ParseFileSize returns // the expected output and error for each. for _, test := range tests { res, err := parseFilesize(test.in) if res != test.out || err != test.err { t.Errorf("parseFilesize(%v): expected %v %v, got %v %v", test.in, test.out, test.err, res, err) } } } ``` ## Questions? Read these if you haven't already: * [doc/Guide to Contributing to Sia.md][guide]: getting started with Go, Sia, and git * [doc/Developers.md][developers]: conventions and quality standards for Sia code Some other useful resources, some of which have been linked to already: * [Golang.org page on the go testing package][pkg/testing] * [Writing Table-Driven Tests in Go][table] * [How to Write Benchmarks in Go][cheney-benchmarks] * [How to into git and GitHub][luke]: an essential introduction to git And feel free to ask questions on the [#dev channel][slack] on the Sia Slack. Odds are, someone else is wondering the same thing. [pkg/testing]: https://golang.org/pkg/testing/ [makefile]: https://github.com/NebulousLabs/Sia/blob/master/Makefile [luke]: https://gist.github.com/lukechampine/6418449 [guide]: https://github.com/NebulousLabs/Sia/blob/master/doc/Guide%20to%20Contributing%20to%20Sia.md [developers]: https://github.com/NebulousLabs/Sia/blob/master/doc/Developers.md [table]: http://dave.cheney.net/2013/06/09/writing-table-driven-tests-in-go [boltdb_test.go]: https://github.com/NebulousLabs/Sia/blob/master/persist/boltdb_test.go [cheney-benchmarks]: http://dave.cheney.net/2013/06/30/how-to-write-benchmarks-in-go [pkg/testing]: https://golang.org/pkg/testing/ [slack]: https://siatalk.slack.com/messages/dev/ [parse_test]: https://github.com/NebulousLabs/Sia/blob/master/siac/parse_test.go [global]: http://c2.com/cgi/wiki?GlobalVariablesAreBad Sia-1.0.4/doc/Standard.md000066400000000000000000000051161302033070500150530ustar00rootroot00000000000000Standard Transaction Rules ========================== Some transactions will not be accepted by miners unless they appear in a block. This is equivalent to the 'IsStandard' function in Bitcoin. This file dictates the rules for standard Sia transactions. Transaction Size ---------------- Consensus rules limit the size of a block, but not the size of a transaction. Standard rules however limit the size of a single transaction to 16kb. A chain of dependent transactions cannot exceed 500kb. Double Spend Rules ------------------ When two conflicting transactions are seen, the first transaction is the only one that is kept. If the blockchain reorganizes, the transaction that is kept is the transaction that was most recently in the blockchain. This is to discourage double spending, and enforce that the first transaction seen is the one that should be kept by the network. Other conflicts are thrown out. Transactions are currently included into blocks using a first-come first-serve algorithm. Eventually, transactions will be rejected if the fee does not meet a certain minimum. For the near future, there are no plans to prioritize transactions with substantially higher fees. Other mining software may take alternative approaches. File Contract Rules ------------------- File Contracts that start in less than 10 blocks time are not accepted into the transaction pool. This is because a file contract becomes invalid if it is not accepted into the blockchain by the start block, and this might result in a cascade of invalidated unconfirmed transactions, which may make it easier to launch double spend attacks on zero confirmation outputs. 10 blocks is plenty of time on the other hand for a file contract to make it into the blockchain. Signature Algorithms -------------------- Miners will reject transactions that have public keys using algorithms that the miner does not understand. Arbitrary Data Usage -------------------- Arbitrary data can be used to make verifiable announcements, or to have other protocols sit on top of Sia. The arbitrary data can also be used for soft forks, and for protocol relevant information. Any arbitrary data is allowed by consensus, but only certain arbitrary data is considered standard. Arbitrary data that is prefixed by the string 'NonSia' is always allowed. This indicates that the remaining data has no relevance to Sia protocol rules, and never will. Arbitrary data that is prefixed by the string 'HostAnnouncement' is allowed, but only if the data within accurately decodes to the HostAnnouncement struct found in modules/hostdb.go, and contains no extra information. Sia-1.0.4/doc/api/000077500000000000000000000000001302033070500135375ustar00rootroot00000000000000Sia-1.0.4/doc/api/Consensus.md000066400000000000000000000027161302033070500160470ustar00rootroot00000000000000Consensus API ============= This document contains detailed descriptions of the consensus's API routes. For an overview of the consensus' API routes, see [API.md#consensus](/doc/API.md#consensus). For an overview of all API routes, see [API.md](/doc/API.md) There may be functional API calls which are not documented. These are not guaranteed to be supported beyond the current release, and should not be used in production. Overview -------- The consensus set manages everything related to consensus and keeps the blockchain in sync with the rest of the network. The consensus set's API endpoint returns information about the state of the blockchain. Index ----- | Route | HTTP verb | | ---------------------------- | --------- | | [/consensus](#consensus-get) | GET | #### /consensus [GET] returns information about the consensus set, such as the current block height. ###### JSON Response ```javascript { // True if the consensus set is synced with the network, i.e. it has downloaded the entire blockchain. "synced": true, // Number of blocks preceding the current block. "height": 62248, // Hash of the current block. "currentblock": "00000000000008a84884ba827bdc868a17ba9c14011de33ff763bd95779a9cf1", // An immediate child block of this block must have a hash less than this // target for it to be valid. "target": [0,0,0,0,0,0,11,48,125,79,116,89,136,74,42,27,5,14,10,31,23,53,226,238,202,219,5,204,38,32,59,165] } ``` Sia-1.0.4/doc/api/Daemon.md000066400000000000000000000067761302033070500153040ustar00rootroot00000000000000Daemon API =========== This document contains detailed descriptions of the daemon's API routes. For an overview of the daemon's API routes, see [API.md#daemon](/doc/API.md#daemon). For an overview of all API routes, see [API.md](/doc/API.md) There may be functional API calls which are not documented. These are not guaranteed to be supported beyond the current release, and should not be used in production. Overview -------- The daemon is responsible for starting and stopping the modules which make up the rest of Sia. It also provides endpoints for viewing build constants. Index ----- | Route | HTTP verb | | ----------------------------------------- | --------- | | [/daemon/constants](#daemonconstants-get) | GET | | [/daemon/stop](#daemonstop-get) | GET | | [/daemon/version](#daemonversion-get) | GET | #### /daemon/constants [GET] returns the set of constants in use. ###### JSON Response ```javascript { // Timestamp of the genesis block. "genesistimestamp": 1433600000, // Unix time // Maximum size, in bytes, of a block. Blocks larger than this will be // rejected by peers. "blocksizelimit": 2000000, // bytes // Target for how frequently new blocks should be mined. "blockfrequency": 600, // seconds per block // Height of the window used to adjust the difficulty. "targetwindow": 1000, // blocks // Duration of the window used to adjust the difficulty. "mediantimestampwindow": 11, // blocks // How far in the future a block can be without being rejected. A block // further into the future will not be accepted immediately, but the daemon // will attempt to accept the block as soon as it is valid. "futurethreshold": 10800, // seconds // Total number of siafunds. "siafundcount": "10000", // Fraction of each file contract payout given to siafund holders. "siafundportion": "39/1000", // Number of children a block must have before it is considered "mature." "maturitydelay": 144, // blocks // Number of coins given to the miner of the first block. Note that elsewhere // in the API currency is typically returned in hastings and as a bignum. // This is not the case here. "initialcoinbase": 300000, // SiaCoins. // Minimum number of coins paid out to the miner of a block (the coinbase // decreases with each block). Note that elsewhere in the API currency is // typically returned in hastings and as a bignum. This is not the case // here. "minimumcoinbase": 30000, // SiaCoins // Initial target. "roottarget": [0,0,0,0,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0], // Initial depth. "rootdepth": [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255], // Largest allowed ratio between the old difficulty and the new difficulty. "maxadjustmentup": "5/2", // Smallest allowed ratio between the old difficulty and the new difficulty. "maxadjustmentdown": "2/5", // Number of Hastings in one siacoin. "siacoinprecision": "1000000000000000000000000" // hastings per siacoin } ``` #### /daemon/stop [GET] cleanly shuts down the daemon. May take a few seconds. ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /daemon/version [GET] returns the version of the Sia daemon currently running. ###### JSON Response ```javascript { // Version number of the running Sia Daemon. This number is visible to its // peers on the network. "version": "1.0.0" } ``` Sia-1.0.4/doc/api/Gateway.md000066400000000000000000000112771302033070500154720ustar00rootroot00000000000000Gateway API =========== This document contains detailed descriptions of the gateway's API routes. For an overview of the gateway's API routes, see [API.md#gateway](/doc/API.md#gateway). For an overview of all API routes, see [API.md](/doc/API.md) There may be functional API calls which are not documented. These are not guaranteed to be supported beyond the current release, and should not be used in production. Overview -------- The gateway maintains a peer to peer connection to the network and provides a method for calling RPCs on connected peers. The gateway's API endpoints expose methods for viewing the connected peers, manually connecting to peers, and manually disconnecting from peers. The gateway may connect or disconnect from peers on its own. Index ----- | Route | HTTP verb | Examples | | ---------------------------------------------------------------------------------- | --------- | ------------------------------------------------------- | | [/gateway](#gateway-get-example) | GET | [Gateway info](#gateway-info) | | [/gateway/connect/___:netaddress___](#gatewayconnectnetaddress-post-example) | POST | [Connecting to a peer](#connecting-to-a-peer) | | [/gateway/disconnect/___:netaddress___](#gatewaydisconnectnetaddress-post-example) | POST | [Disconnecting from a peer](#disconnecting-from-a-peer) | #### /gateway [GET] [(example)](#gateway-info) returns information about the gateway, including the list of connected peers. ###### JSON Response ```javascript { // netaddress is the network address of the gateway as seen by the rest of // the network. The address consists of the external IP address and the // port Sia is listening on. It represents a `modules.NetAddress`. "netaddress": String, // peers is an array of peers the gateway is connected to. It represents // an array of `modules.Peer`s. "peers": []{ // netaddress is the address of the peer. It represents a // `modules.NetAddress`. "netaddress": String, // version is the version number of the peer. "version": String, // inbound is true when the peer initiated the connection. This field // is exposed as outbound peers are generally trusted more than inbound // peers, as inbound peers are easily manipulated by an adversary. "inbound": Boolean } } ``` #### /gateway/connect/{netaddress} [POST] [(example)](#connecting-to-a-peer) connects the gateway to a peer. The peer is added to the node list if it is not already present. The node list is the list of all nodes the gateway knows about, but is not necessarily connected to. ###### Path Parameters ``` // netaddress is the address of the peer to connect to. It should be a // reachable ip address and port number, of the form 'IP:port'. IPV6 addresses // must be enclosed in square brackets. // // Example IPV4 address: 123.456.789.0:123 // Example IPV6 address: [123::456]:789 :netaddress ``` ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). #### /gateway/disconnect/{netaddress} [POST] [(example)](#disconnecting-from-a-peer) disconnects the gateway from a peer. The peer remains in the node list. Disconnecting from a peer does not prevent the gateway from automatically connecting to the peer in the future. ###### Path Parameters ``` // netaddress is the address of the peer to connect to. It should be a // reachable ip address and port number, of the form 'IP:port'. IPV6 addresses // must be enclosed in square brackets. // // Example IPV4 address: 123.456.789.0:123 // Example IPV6 address: [123::456]:789 :netaddress ``` ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). Examples -------- #### Gateway info ###### Request ``` /gateway ``` ###### Expected Response Code ``` 200 OK ``` ###### Example JSON Response ```json { "netaddress":"333.333.333.333:9981", "peers":[ { "netaddress":"222.222.222.222:9981", "version":"1.0.0", "inbound":false }, { "netaddress":"111.111.111.111:9981", "version":"0.6.0", "inbound":true } ] } ``` #### Connecting to a peer ###### Request ``` /gateway/connect/123.456.789.0:123 ``` ###### Expected Response Code ``` 204 No Content ``` #### Disconnecting from a peer ###### Request ``` /gateway/disconnect/123.456.789.0:123 ``` ###### Expected Response Code ``` 204 No Content ``` Sia-1.0.4/doc/api/Host.md000066400000000000000000000542021302033070500150010ustar00rootroot00000000000000Host API -------- This document contains detailed descriptions of the host's API routes. For an overview of the host's API routes, see [API.md#host](/doc/API.md#host). For an overview of all API routes, see [API.md](/doc/API.md) There may be functional API calls which are not documented. These are not guaranteed to be supported beyond the current release, and should not be used in production. Overview -------- The host provides storage from local disks to the network. The host negotiates file contracts with remote renters to earn money for storing other users' files. The host's endpoints expose methods for viewing and modifying host settings, announcing to the network, and managing how files are stored on disk. Index ----- | Route | HTTP verb | | ------------------------------------------------------------------------------------- | --------- | | [/host](#host-get) | GET | | [/host](#host-post) | POST | | [/host/announce](#hostannounce-post) | POST | | [/host/storage](#hoststorage-get) | GET | | [/host/storage/folders/add](#hoststoragefoldersadd-post) | POST | | [/host/storage/folders/remove](#hoststoragefoldersremove-post) | POST | | [/host/storage/folders/resize](#hoststoragefoldersresize-post) | POST | | [/host/storage/sectors/delete/___:merkleroot___](#hoststoragesectorsdeletemerkleroot) | POST | #### /host [GET] fetches status information about the host. ###### JSON Response ```javascript { // The settings that get displayed to untrusted nodes querying the host's // status. "externalsettings": { // Whether or not the host is accepting new contracts. "acceptingcontracts": true, // The maximum size of a single download request from a renter. Each // download request has multiple round trips of communication that // exchange money. Larger batch sizes mean fewer round trips, but more // financial risk for the host - the renter can get a free batch when // downloading by refusing to provide a signature. "maxdownloadbatchsize": 17825792, // bytes // The maximum duration that a host will allow for a file contract. The // host commits to keeping files for the full duration under the threat // of facing a large penalty for losing or dropping data before the // duration is complete. The storage proof window of an incoming file // contract must end before the current height + maxduration. "maxduration": 25920, // blocks // The maximum size of a single batch of file contract revisions. The // renter can perform DoS attacks on the host by uploading a batch of // data then refusing to provide a signature to pay for the data. The // host can reduce this exposure by limiting the batch size. Larger // batch sizes allow for higher throughput as there is significant // communication overhead associated with performing a batch upload. "maxrevisebatchsize": 17825792, // bytes // The IP address or hostname (including port) that the host should be // contacted at. "netaddress": "123.456.789.0:9982", // The amount of unused storage capacity on the host in bytes. It // should be noted that the host can lie. "remainingstorage": 35000000000, // bytes // The smallest amount of data in bytes that can be uploaded or // downloaded when performing calls to the host. "sectorsize": 4194304, // bytes // The total amount of storage capacity on the host. It should be noted // that the host can lie. "totalstorage": 35000000000, // bytes // The unlock hash is the address at which the host can be paid when // forming file contracts. "unlockhash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab", // The storage proof window is the number of blocks that the host has // to get a storage proof onto the blockchain. The window size is the // minimum size of window that the host will accept in a file contract. "windowsize": 144, // blocks // The maximum amount of money that the host will put up as collateral // for storage that is contracted by the renter. "collateral": "57870370370", // hastings / byte / block // The maximum amount of collateral that the host will put into a // single file contract. "maxcollateral": "100000000000000000000000000000", // hastings // The price that a renter has to pay to create a contract with the // host. The payment is intended to cover transaction fees // for the file contract revision and the storage proof that the host // will be submitting to the blockchain. "contractprice": "30000000000000000000000000", // hastings // The price that a renter has to pay when downloading data from the // host. "downloadbandwidthprice": "250000000000000", // hastings / byte // The price that a renter has to pay to store files with the host. "storageprice": "231481481481", // hastings / byte / block // The price that a renter has to pay when uploading data to the host. "uploadbandwidthprice": "100000000000000", // hastings / byte // The revision number indicates to the renter what iteration of // settings the host is currently at. Settings are generally signed. // If the renter has multiple conflicting copies of settings from the // host, the renter can expect the one with the higher revision number // to be more recent. "revisionnumber": 0, // The version of external settings being used. This field helps // coordinate updates while preserving compatibility with older nodes. "version": "1.0.0" }, // The financial status of the host. "financialmetrics": { // Number of open file contracts. "contractcount": 2, // The amount of money that renters have given to the host to pay for // file contracts. The host is required to submit a file contract // revision and a storage proof for every file contract that gets created, // and the renter pays for the miner fees on these objects. "contractcompensation": "123", // hastings // The amount of money that renters have given to the host to pay for // file contracts which have not been confirmed yet. The potential // compensation becomes compensation after the storage proof is // submitted. "potentialcontractcompensation": "123", // hastings // The amount of storage collateral which the host has tied up in file // contracts. The host has to commit collateral to a file contract even // if there is no storage, but the locked collateral will be returned // even if the host does not submit a storage proof - the collateral is // not at risk, it is merely set aside so that it can be put at risk // later. "lockedstoragecollateral": "123", // hastings // The amount of revenue, including storage revenue and bandwidth // revenue, that has been lost due to failed file contracts and // failed storage proofs. "lostrevenue": "123", // hastings // The amount of collateral that was put up to protect data which has // been lost due to failed file contracts and missed storage proofs. "loststoragecollateral": "123", // hastings // The amount of revenue that the host stands to earn if all storage // proofs are submitted corectly and in time. "potentialstoragerevenue": "123", // hastings // The amount of money that the host has risked on file contracts. If // the host starts missing storage proofs, the host can forfeit up to // this many coins. In the event of a missed storage proof, locked // storage collateral gets returned, but risked storage collateral // does not get returned. "riskedstoragecollateral": "123", // hastings // The amount of money that the host has earned from storing data. This // money has been locked down by successful storage proofs. "storagerevenue": "123", // hastings // The amount of money that the host has spent on transaction fees when // submitting host announcements, file contract revisions, and storage // proofs. "transactionfeeexpenses": "123", // hastings // The amount of money that the host has made from renters downloading // their files. This money has been locked in by successsful storage // proofs. "downloadbandwidthrevenue": "123", // hastings // The amount of money that the host stands to make from renters that // downloaded their files. The host will only realize this revenue if // the host successfully submits storage proofs for the related file // contracts. "potentialdownloadbandwidthrevenue": "123", // hastings // The amount of money that the host stands to make from renters that // uploaded files. The host will only realize this revenue if the host // successfully submits storage proofs for the related file contracts. "potentialuploadbandwidthrevenue": "123", // hastings // The amount of money that the host has made from renters uploading // their files. This money has been locked in by successful storage // proofs. "uploadbandwidthrevenue": "123" // hastings }, // The settings of the host. Most interactions between the user and the // host occur by changing the internal settings. "internalsettings": { // When set to true, the host will accept new file contracts if the // terms are reasonable. When set to false, the host will not accept new // file contracts at all. "acceptingcontracts": true, // The maximum size of a single download request from a renter. Each // download request has multiple round trips of communication that // exchange money. Larger batch sizes mean fewer round trips, but more // financial risk for the host - the renter can get a free batch when // downloading by refusing to provide a signature. "maxdownloadbatchsize": 17825792, // bytes // The maximum duration of a file contract that the host will accept. // The storage proof window must end before the current height + // maxduration. "maxduration": 25920, // blocks // The maximum size of a single batch of file contract revisions. The // renter can perform DoS attacks on the host by uploading a batch of // data then refusing to provide a signature to pay for the data. The // host can reduce this exposure by limiting the batch size. Larger // batch sizes allow for higher throughput as there is significant // communication overhead associated with performing a batch upload. "maxrevisebatchsize": 17825792, // bytes // The IP address or hostname (including port) that the host should be // contacted at. If left blank, the host will automatically figure out // its ip address and use that. If given, the host will use the address // given. "netaddress": "123.456.789.0:9982", // The storage proof window is the number of blocks that the host has // to get a storage proof onto the blockchain. The window size is the // minimum size of window that the host will accept in a file contract. "windowsize": 144, // blocks // The maximum amount of money that the host will put up as collateral // per byte per block of storage that is contracted by the renter. "collateral": "57870370370", // hastings / byte / block // The total amount of money that the host will allocate to collateral // across all file contracts. "collateralbudget": "2000000000000000000000000000000", // hastings // The maximum amount of collateral that the host will put into a // single file contract. "maxcollateral": "100000000000000000000000000000", // hastings // The minimum price that the host will demand from a renter when // forming a contract. Typically this price is to cover transaction // fees on the file contract revision and storage proof, but can also // be used if the host has a low amount of collateral. The price is a // minimum because the host may automatically adjust the price upwards // in times of high demand. "mincontractprice": "30000000000000000000000000", // hastings // The minimum price that the host will demand from a renter when the // renter is downloading data. If the host is saturated, the host may // increase the price from the minimum. "mindownloadbandwidthprice": "250000000000000", // hastings / byte // The minimum price that the host will demand when storing data for // extended periods of time. If the host is low on space, the price of // storage may be set higher than the minimum. "minstorageprice": "231481481481", // hastings / byte / block // The minimum price that the host will demand from a renter when the // renter is uploading data. If the host is saturated, the host may // increase the price from the minimum. "minuploadbandwidthprice": "100000000000000" // hastings / byte }, // Information about the network, specifically various ways in which // renters have contacted the host. "networkmetrics": { // The number of times that a renter has attempted to download // something from the host. "downloadcalls": 0, // The number of calls that have resulted in errors. A small number of // errors are expected, but a large number of errors indicate either // buggy software or malicious network activity. Usually buggy // software. "errorcalls": 1, // The number of times that a renter has tried to form a contract with // the host. "formcontractcalls": 2, // The number of times that a renter has tried to renew a contract with // the host. "renewcalls": 3, // The number of times that the renter has tried to revise a contract // with the host. "revisecalls": 4, // The number of times that a renter has queried the host for the // host's settings. The settings include the price of bandwidth, which // is a price that can adjust every few minutes. This value is usually // very high compared to the others. "settingscalls": 5, // The number of times that a renter has attempted to use an // unrecognized call. Larger numbers typically indicate buggy software. "unrecognizedcalls": 6 } } ``` #### /host [POST] configures hosting parameters. All parameters are optional; unspecified parameters will be left unchanged. ###### Query String Parameters ``` // When set to true, the host will accept new file contracts if the // terms are reasonable. When set to false, the host will not accept new // file contracts at all. acceptingcontracts // Optional, true / false // The maximum size of a single download request from a renter. Each // download request has multiple round trips of communication that // exchange money. Larger batch sizes mean fewer round trips, but more // financial risk for the host - the renter can get a free batch when // downloading by refusing to provide a signature. maxdownloadbatchsize // Optional, bytes // The maximum duration of a file contract that the host will accept. // The storage proof window must end before the current height + // maxduration. maxduration // Optional, blocks // The maximum size of a single batch of file contract revisions. The // renter can perform DoS attacks on the host by uploading a batch of // data then refusing to provide a signature to pay for the data. The // host can reduce this exposure by limiting the batch size. Larger // batch sizes allow for higher throughput as there is significant // communication overhead associated with performing a batch upload. maxrevisebatchsize // Optional, bytes // The IP address or hostname (including port) that the host should be // contacted at. If left blank, the host will automatically figure out // its ip address and use that. If given, the host will use the address // given. netaddress // Optional // The storage proof window is the number of blocks that the host has // to get a storage proof onto the blockchain. The window size is the // minimum size of window that the host will accept in a file contract. windowsize // Optional, blocks // The maximum amount of money that the host will put up as collateral // per byte per block of storage that is contracted by the renter. collateral // Optional, hastings / byte / block // The total amount of money that the host will allocate to collateral // across all file contracts. collateralbudget // Optional, hastings // The maximum amount of collateral that the host will put into a // single file contract. maxcollateral // Optional, hastings // The minimum price that the host will demand from a renter when // forming a contract. Typically this price is to cover transaction // fees on the file contract revision and storage proof, but can also // be used if the host has a low amount of collateral. The price is a // minimum because the host may automatically adjust the price upwards // in times of high demand. mincontractprice // Optional, hastings // The minimum price that the host will demand from a renter when the // renter is downloading data. If the host is saturated, the host may // increase the price from the minimum. mindownloadbandwidthprice // Optional, hastings / byte // The minimum price that the host will demand when storing data for // extended periods of time. If the host is low on space, the price of // storage may be set higher than the minimum. minstorageprice // Optional, hastings / byte / block // The minimum price that the host will demand from a renter when the // renter is uploading data. If the host is saturated, the host may // increase the price from the minimum. minuploadbandwidthprice // Optional, hastings / byte ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /host/announce [POST] Announce the host to the network as a source of storage. Generally only needs to be called once. ###### Query String Parameters ``` // The address to be announced. If no address is provided, the automatically // discovered address will be used instead. netaddress string // Optional ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /host/storage [GET] gets a list of folders tracked by the host's storage manager. ###### JSON Response ```javascript { "folders": [ { // Absolute path to the storage folder on the local filesystem. "path": "/home/foo/bar", // Maximum capacity of the storage folder. The host will not store more // than this many bytes in the folder. This capacity is not checked // against the drive's remaining capacity. Therefore, you must manually // ensure the disk has sufficient capacity for the folder at all times. // Otherwise you risk losing renter's data and failing storage proofs. "capacity": 50000000000, // bytes // Unused capacity of the storage folder. "capacityremaining": 100000, // bytes // Number of failed disk read & write operations. A large number of // failed reads or writes indicates a problem with the filesystem or // drive's hardware. "failedreads": 0, "failedwrites": 1, // Number of successful read & write operations. "successfulreads": 2, "successfulwrites": 3 } ] } ``` #### /host/storage/folders/add [POST] adds a storage folder to the manager. The manager may not check that there is enough space available on-disk to support as much storage as requested ###### Query String Parameters ``` // Local path on disk to the storage folder to add. path // Required // Initial capacity of the storage folder. This value isn't validated so it is // possible to set the capacity of the storage folder greater than the capacity // of the disk. Do not do this. size // bytes, Required ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /host/storage/folders/remove [POST] remove a storage folder from the manager. All storage on the folder will be moved to other storage folders, meaning that no data will be lost. If the manager is unable to save data, an error will be returned and the operation will be stopped. ###### Query String Parameters ``` // Local path on disk to the storage folder to remove. path // Required // If `force` is true, the storage folder will be removed even if the data in // the storage folder cannot be moved to other storage folders, typically // because they don't have sufficient capacity. If `force` is true and the data // cannot be moved, data will be lost. force // bool, Optional, default is false ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /host/storage/folders/resize [POST] grows or shrink a storage folder in the manager. The manager may not check that there is enough space on-disk to support growing the storage folder, but should gracefully handle running out of space unexpectedly. When shrinking a storage folder, any data in the folder that needs to be moved will be placed into other storage folders, meaning that no data will be lost. If the manager is unable to migrate the data, an error will be returned and the operation will be stopped. ###### Query String Parameters ``` // Local path on disk to the storage folder to resize. path // Required // Desired new size of the storage folder. This will be the new capacity of the // storage folder. newsize // bytes, Required ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). #### /host/storage/sectors/delete/___*merkleroot___ [POST] deletes a sector, meaning that the manager will be unable to upload that sector and be unable to provide a storage proof on that sector. This endpoint is for removing the data entirely, and will remove instances of the sector appearing at all heights. The primary purpose is to comply with legal requests to remove data. ###### Path Parameters ``` // Merkleroot of the sector to delete. :merkleroot ``` ###### Response standard success or error response. See [#standard-responses](#standard-responses). Sia-1.0.4/doc/api/HostDB.md000066400000000000000000000222341302033070500152070ustar00rootroot00000000000000Host DB API =========== This document contains detailed descriptions of the hostdb's API routes. For an overview of the hostdb's API routes, see [API.md#host-db](/doc/API.md#host-db). For an overview of all API routes, see [API.md](/doc/API.md) There may be functional API calls which are not documented. These are not guaranteed to be supported beyond the current release, and should not be used in production. Overview -------- The hostdb maintains a database of all hosts known to the network. The database identifies hosts by their public key and keeps track of metrics such as price. Index ----- | Request | HTTP Verb | Examples | | ------------------------------------------- | --------- | ----------------------------- | | [/hostdb/active](#hostdbactive-get-example) | GET | [Active hosts](#active-hosts) | | [/hostdb/all](#hostdball-get-example) | GET | [All hosts](#all-hosts) | #### /hostdb/active [GET] [(example)](#active-hosts) lists all of the active hosts known to the renter, sorted by preference. ###### Query String Parameters ``` // Number of hosts to return. The actual number of hosts returned may be less // if there are insufficient active hosts. Optional, the default is all active // hosts. numhosts ``` ###### JSON Response ```javascript { "hosts": [ { // true if the host is accepting new contracts. "acceptingcontracts": true, // Maximum number of bytes that the host will allow to be requested by a // single download request. "maxdownloadbatchsize": 17825792, // Maximum duration in blocks that a host will allow for a file contract. // The host commits to keeping files for the full duration under the // threat of facing a large penalty for losing or dropping data before // the duration is complete. The storage proof window of an incoming file // contract must end before the current height + maxduration. // // There is a block approximately every 10 minutes. // e.g. 1 day = 144 blocks "maxduration": 25920, // Maximum size in bytes of a single batch of file contract // revisions. Larger batch sizes allow for higher throughput as there is // significant communication overhead associated with performing a batch // upload. "maxrevisebatchsize": 17825792, // Remote address of the host. It can be an IPv4, IPv6, or hostname, // along with the port. IPv6 addresses are enclosed in square brackets. "netaddress": "123.456.789.0:9982", // Unused storage capacity the host claims it has, in bytes. "remainingstorage": 35000000000, // Smallest amount of data in bytes that can be uploaded or downloaded to // or from the host. "sectorsize": 4194304, // Total amount of storage capacity the host claims it has, in bytes. "totalstorage": 35000000000, // Address at which the host can be paid when forming file contracts. "unlockhash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab", // A storage proof window is the number of blocks that the host has to // get a storage proof onto the blockchain. The window size is the // minimum size of window that the host will accept in a file contract. "windowsize": 144, // Public key used to identify and verify hosts. "publickey": { // Algorithm used for signing and verification. Typically "ed25519". "algorithm": "ed25519", // Key used to verify signed host messages. "key": "RW50cm9weSBpc24ndCB3aGF0IGl0IHVzZWQgdG8gYmU=" } } ] } ``` #### /hostdb/all [GET] [(example)](#all-hosts) lists all of the hosts known to the renter. Hosts are not guaranteed to be in any particular order, and the order may change in subsequent calls. ###### JSON Response ```javascript { "hosts": [ { // true if the host is accepting new contracts. "acceptingcontracts": true, // Maximum number of bytes that the host will allow to be requested by a // single download request. "maxdownloadbatchsize": 17825792, // Maximum duration in blocks that a host will allow for a file contract. // The host commits to keeping files for the full duration under the // threat of facing a large penalty for losing or dropping data before // the duration is complete. The storage proof window of an incoming file // contract must end before the current height + maxduration. // // There is a block approximately every 10 minutes. // e.g. 1 day = 144 blocks "maxduration": 25920, // Maximum size in bytes of a single batch of file contract // revisions. Larger batch sizes allow for higher throughput as there is // significant communication overhead associated with performing a batch // upload. "maxrevisebatchsize": 17825792, // Remote address of the host. It can be an IPv4, IPv6, or hostname, // along with the port. IPv6 addresses are enclosed in square brackets. "netaddress": "123.456.789.0:9982", // Unused storage capacity the host claims it has, in bytes. "remainingstorage": 35000000000, // Smallest amount of data in bytes that can be uploaded or downloaded to // or from the host. "sectorsize": 4194304, // Total amount of storage capacity the host claims it has, in bytes. "totalstorage": 35000000000, // Address at which the host can be paid when forming file contracts. "unlockhash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab", // A storage proof window is the number of blocks that the host has to // get a storage proof onto the blockchain. The window size is the // minimum size of window that the host will accept in a file contract. "windowsize": 144, // Public key used to identify and verify hosts. "publickey": { // Algorithm used for signing and verification. Typically "ed25519". "algorithm": "ed25519", // Key used to verify signed host messages. "key": "RW50cm9weSBpc24ndCB3aGF0IGl0IHVzZWQgdG8gYmU=" } } ] } ``` Examples -------- #### Active hosts ###### Request ``` /hostdb/active?numhosts=2 ``` ###### Expected Response Code ``` 200 OK ``` ###### Example JSON Response ```javascript { "hosts": [ { "acceptingcontracts": true, "maxdownloadbatchsize": 17825792, "maxduration": 25920, "maxrevisebatchsize": 17825792, "netaddress": "123.456.789.0:9982", "remainingstorage": 35000000000, "sectorsize": 4194304, "totalstorage": 35000000000, "unlockhash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab", "windowsize": 144, "publickey": { "algorithm": "ed25519", "key": "RW50cm9weSBpc24ndCB3aGF0IGl0IHVzZWQgdG8gYmU=" } }, { "acceptingcontracts": true, "maxdownloadbatchsize": 17825792, "maxduration": 25920, "maxrevisebatchsize": 17825792, "netaddress": "123.456.789.1:9982", "remainingstorage": 314, "sectorsize": 4194304, "totalstorage": 314159265359, "unlockhash": "ba9876543210fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", "windowsize": 144, "publickey": { "algorithm": "ed25519", "key": "WWVzIEJydWNlIFNjaG5laWVyIGNhbiByZWFkIHRoaXM=" } } ] } ``` #### All hosts ###### Request ``` /hostdb/all ``` ###### Expected Response Code ``` 200 OK ``` ###### Example JSON Response ```javascript { "hosts": [ { "acceptingcontracts": false, "maxdownloadbatchsize": 17825792, "maxduration": 25920, "maxrevisebatchsize": 17825792, "netaddress": "123.456.789.2:9982", "remainingstorage": 314, "sectorsize": 4194304, "totalstorage": 314159265359, "unlockhash": "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", "windowsize": 144, "publickey": { "algorithm": "ed25519", "key": "SSByYW4gb3V0IG9mIDMyIGNoYXIgbG9uZyBqb2tlcy4=" } }, { "acceptingcontracts": true, "maxdownloadbatchsize": 17825792, "maxduration": 25920, "maxrevisebatchsize": 17825792, "netaddress": "123.456.789.0:9982", "remainingstorage": 35000000000, "sectorsize": 4194304, "totalstorage": 35000000000, "unlockhash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab", "windowsize": 144, "publickey": { "algorithm": "ed25519", "key": "RW50cm9weSBpc24ndCB3aGF0IGl0IHVzZWQgdG8gYmU=" } }, { "acceptingcontracts": true, "maxdownloadbatchsize": 17825792, "maxduration": 25920, "maxrevisebatchsize": 17825792, "netaddress": "123.456.789.1:9982", "remainingstorage": 314, "sectorsize": 4194304, "totalstorage": 314159265359, "unlockhash": "ba9876543210fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", "windowsize": 144, "publickey": { "algorithm": "ed25519", "key": "WWVzIEJydWNlIFNjaG5laWVyIGNhbiByZWFkIHRoaXM=" } } ] } ``` Sia-1.0.4/doc/api/Miner.md000066400000000000000000000115761302033070500151450ustar00rootroot00000000000000Miner API ========= This document contains detailed descriptions of the miner's API routes. For an overview of the miner's API routes, see [API.md#miner](/doc/API.md#miner). For an overview of all API routes, see [API.md](/doc/API.md) There may be functional API calls which are not documented. These are not guaranteed to be supported beyond the current release, and should not be used in production. Overview -------- The miner provides endpoints for getting headers for work and submitting solved headers to the network. The miner also provides endpoints for controlling a basic CPU mining implementation. Index ----- | Route | HTTP verb | | ---------------------------------- | --------- | | [/miner](#miner-get) | GET | | [/miner/start](#minerstart-get) | GET | | [/miner/stop](#minerstop-get) | GET | | [/miner/header](#minerheader-get) | GET | | [/miner/header](#minerheader-post) | POST | #### /miner [GET] returns the status of the miner. ###### JSON Response ```javascript { // Number of mined blocks. This value is remembered after restarting. "blocksmined": 9001, // How fast the cpu is hashing, in hashes per second. "cpuhashrate": 1337, // true if the cpu miner is active. "cpumining": false, // Number of mined blocks that are stale, indicating that they are not // included in the current longest chain, likely because some other block at // the same height had its chain extended first. "staleblocksmined": 0, } ``` #### /miner/start [GET] starts a single threaded cpu miner. Does nothing if the cpu miner is already running. ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). #### /miner/stop [GET] stops the cpu miner. Does nothing if the cpu miner is not running. ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). #### /miner/header [GET] provides a block header that is ready to be grinded on for work. ###### Byte Response For efficiency the header for work is returned as a raw byte encoding of the header, rather than encoded to JSON. Blocks are mined by repeatedly changing the nonce of the header, hashing the header's bytes, and comparing the resulting hash to the target. The block with that nonce is valid if the hash is less than the target. If none of the 2^64 possible nonces result in a header with a hash less than the target, call `/miner/header [GET]` again to get a new block header with a different merkle root. The above process can then be repeated for the new block header. The other fields can generally be ignored. The parent block ID field is the hash of the parent block's header. Modifying this field will result in an orphan block. The timestamp is the time at which the block was mined and is set by the Sia Daemon. Modifying this field can result in invalid block. The merkle root is the merkle root of a merkle tree consisting of the timestamp, the miner outputs (one leaf per payout), and the transactions (one leaf per transaction). Modifying this field will result in an invalid block. | Field | Byte range within response | Byte range within header | | --------------- | -------------------------- | ------------------------ | | target | [0-32) | | | header | [32-112) | | | parent block ID | [32-64) | [0-32) | | nonce | [64-72) | [32-40) | | timestamp | [72-80) | [40-48) | | merkle root | [80-112) | [48-80) | ``` xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx (returned bytes) tttttttttttttttttttttttttttttttt (target) hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh (header) pppppppppppppppppppppppppppppppp (parent block ID) nnnnnnnn (nonce) ssssssss (timestamp) mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm (merkle root) ``` #### /miner/header [POST] submits a header that has passed the POW. ###### Request Body Bytes For efficiency headers are submitted as raw byte encodings of the header in the body of the request, rather than as a query string parameter or path parameter. The request body should contain only the 80 bytes of the encoded header. The encoding is the same encoding used in `/miner/header [GET]` endpoint. Refer to [#byte-response](#byte-response) for a detailed description of the byte encoding. Sia-1.0.4/doc/api/Renter.md000066400000000000000000000175651302033070500153360ustar00rootroot00000000000000Renter API ========== This document contains detailed descriptions of the renter's API routes. For an overview of the renter's API routes, see [API.md#renter](/doc/API.md#renter). For an overview of all API routes, see [API.md](/doc/API.md) There may be functional API calls which are not documented. These are not guaranteed to be supported beyond the current release, and should not be used in production. Overview -------- The renter manages the user's files on the network. The renter's API endpoints expose methods for managing files on the network and managing the renter's allocated funds. Index ----- | Route | HTTP verb | | ------------------------------------------------------------- | --------- | | [/renter](#renter-get) | GET | | [/renter](#renter-post) | POST | | [/renter/contracts](#rentercontracts-get) | GET | | [/renter/downloads](#renterdownloads-get) | GET | | [/renter/files](#renterfiles-get) | GET | | [/renter/delete/___*siapath___](#renterdeletesiapath-post) | POST | | [/renter/download/___*siapath___](#renterdownloadsiapath-get) | GET | | [/renter/rename/___*siapath___](#renterrenamesiapath-post) | POST | | [/renter/upload/___*siapath___](#renteruploadsiapath-post) | POST | #### /renter [GET] returns the current settings along with metrics on the renter's spending. ###### JSON Response ```javascript { // Settings that control the behavior of the renter. "settings": { // Allowance dictates how much the renter is allowed to spend in a given // period. Note that funds are spent on both storage and bandwidth. "allowance": { // Amount of money allocated for contracts. Funds are spent on both // storage and bandwidth. "funds": "1234", // hastings // Number of hosts that contracts will be formed with. "hosts":24, // Duration of contracts formed, in number of blocks. "period": 6048, // blocks // If the current blockheight + the renew window >= the height the // contract is scheduled to end, the contract is renewed automatically. // Is always nonzero. "renewwindow": 3024 // blocks } }, // Metrics about how much the Renter has spent on storage, uploads, and // downloads. "financialmetrics": { // How much money, in hastings, the Renter has paid into file contracts // formed with hosts. Note that some of this money may be returned to the // Renter when the contract ends. To calculate how much will be returned, // subtract the storage, upload, and download metrics from // ContractSpending. "contractspending": "1234", // hastings // Amount of money spent on downloads. "downloadspending": "5678", // hastings // Amount of money spend on storage. "storagespending": "1234", // hastings // Amount of money spent on uploads. "uploadspending": "5678" // hastings } } ``` #### /renter [POST] modify settings that control the renter's behavior. ###### Query String Parameters ``` // Number of hastings allocated for file contracts in the given period. funds // hastings // Duration of contracts formed. Must be nonzero. period // block height ``` ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). #### /renter/contracts [GET] returns active contracts. Expired contracts are not included. ###### JSON Response ```javascript { "contracts": [ { // Block height that the file contract ends on. "endheight": 50000, // block height // ID of the file contract. "id": "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef", // Address of the host the file contract was formed with. "netaddress": "12.34.56.78:9", // Remaining funds left for the renter to spend on uploads & downloads. "renterfunds": "1234", // hastings // Size of the file contract, which is typically equal to the number of // bytes that have been uploaded to the host. "size": 8192 // bytes } ] } ``` #### /renter/downloads [GET] lists all files in the download queue. ###### JSON Response ```javascript { "downloads": [ { // Siapath given to the file when it was uploaded. "siapath": "foo/bar.txt", // Local path that the file will be downloaded to. "destination": "/home/users/alice", // Size, in bytes, of the file being downloaded. "filesize": 8192, // bytes // Number of bytes downloaded thus far. "received": 4096, // bytes // Time at which the download was initiated. "starttime": "2009-11-10T23:00:00Z" // RFC 3339 time } ] } ``` #### /renter/files [GET] lists the status of all files. ###### JSON Response ```javascript { "files": [ { // Path to the file in the renter on the network. "siapath": "foo/bar.txt", // Size of the file in bytes. "filesize": 8192, // bytes // true if the file is available for download. Files may be available // before they are completely uploaded. "available": true, // true if the file's contracts will be automatically renewed by the // renter. "renewing": true, // Average redundancy of the file on the network. Redundancy is // calculated by dividing the amount of data uploaded in the file's open // contracts by the size of the file. Redundancy does not necessarily // correspond to availability. Specifically, a redundancy >= 1 does not // indicate the file is available as there could be a chunk of the file // with 0 redundancy. "redundancy": 5, // Percentage of the file uploaded, including redundancy. Uploading has // completed when uploadprogress is 100. Files may be available for // download before upload progress is 100. "uploadprogress": 100, // percent // Block height at which the file ceases availability. "expiration": 60000 } ] } ``` #### /renter/delete/___*siapath___ [POST] deletes a renter file entry. Does not delete any downloads or original files, only the entry in the renter. ###### Path Parameters ``` // Location of the file in the renter on the network. *siapath ``` ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). #### /renter/download/___*siapath___ [GET] downloads a file to the local filesystem. The call will block until the file has been downloaded. ###### Path Parameters ``` // Location of the file in the renter on the network. *siapath ``` ###### Query String Parameters ``` // Location on disk that the file will be downloaded to. destination ``` ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). #### /renter/rename/___*siapath___ [POST] renames a file. Does not rename any downloads or source files, only renames the entry in the renter. An error is returned if `siapath` does not exist or `newsiapath` already exists. ###### Path Parameters ``` // Current location of the file in the renter on the network. *siapath ``` ###### Query String Parameters ``` // New location of the file in the renter on the network. newsiapath ``` ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). #### /renter/upload/___*siapath___ [POST] uploads a file to the network from the local filesystem. ###### Path Parameters ``` // Location where the file will reside in the renter on the network. *siapath ``` ###### Query String Parameters ``` // Location on disk of the file being uploaded. source ``` ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). Sia-1.0.4/doc/api/Wallet.md000066400000000000000000000462511302033070500153210ustar00rootroot00000000000000Wallet ====== This document contains detailed descriptions of the wallet's API routes. For an overview of the wallet's API routes, see [API.md#wallet](/doc/API.md#wallet). For an overview of all API routes, see [API.md](/doc/API.md) There may be functional API calls which are not documented. These are not guaranteed to be supported beyond the current release, and should not be used in production. Overview -------- The wallet stores and manages siacoins and siafunds. The wallet's API endpoints expose methods for creating and loading wallets, locking and unlocking, sending siacoins and siafunds, and getting the wallet's balance. You must create a wallet before you can use the wallet's API endpoints. You can create a wallet with the `/wallet/init` endpoint. Wallets are always encrypted on disk. Calls to some wallet API endpoints will fail until the wallet is unlocked. The wallet can be unlocked with the `/wallet/unlock` endpoint. Once the wallet is unlocked calls to the API endpoints will succeed until the wallet is locked again with `/wallet/lock`, or Siad is restarted. The host and renter require the miner to be unlocked. Index ----- | Route | HTTP verb | | --------------------------------------------------------------- | --------- | | [/wallet](#wallet-get) | GET | | [/wallet/033x](#wallet033x-post) | POST | | [/wallet/address](#walletaddress-get) | GET | | [/wallet/addresses](#walletaddresses-get) | GET | | [/wallet/backup](#walletbackup-get) | GET | | [/wallet/init](#walletinit-post) | POST | | [/wallet/lock](#walletlock-post) | POST | | [/wallet/seed](#walletseed-post) | POST | | [/wallet/seeds](#walletseeds-get) | GET | | [/wallet/siacoins](#walletsiacoins-post) | POST | | [/wallet/siafunds](#walletsiafunds-post) | POST | | [/wallet/siagkey](#walletsiagkey-post) | POST | | [/wallet/transaction/___:id___](#wallettransactionid-get) | GET | | [/wallet/transactions](#wallettransactions-get) | GET | | [/wallet/transactions/___:addr___](#wallettransactionsaddr-get) | GET | | [/wallet/unlock](#walletunlock-post) | POST | #### /wallet [GET] returns basic information about the wallet, such as whether the wallet is locked or unlocked. ###### JSON Response ```javascript { // Indicates whether the wallet has been encrypted or not. If the wallet // has not been encrypted, then no data has been generated at all, and the // first time the wallet is unlocked, the password given will be used as // the password for encrypting all of the data. 'encrypted' will only be // set to false if the wallet has never been unlocked before (the unlocked // wallet is still encryped - but the encryption key is in memory). "encrypted": true, // Indicates whether the wallet is currently locked or unlocked. Some calls // become unavailable when the wallet is locked. "unlocked": true, // Number of siacoins, in hastings, available to the wallet as of the most // recent block in the blockchain. "confirmedsiacoinbalance": "123456", // hastings, big int // Number of siacoins, in hastings, that are leaving the wallet according // to the set of unconfirmed transactions. Often this number appears // inflated, because outputs are frequently larger than the number of coins // being sent, and there is a refund. These coins are counted as outgoing, // and the refund is counted as incoming. The difference in balance can be // calculated using 'unconfirmedincomingsiacoins' - 'unconfirmedoutgoingsiacoins' "unconfirmedoutgoingsiacoins": "0", // hastings, big int // Number of siacoins, in hastings, are entering the wallet according to // the set of unconfirmed transactions. This number is often inflated by // outgoing siacoins, because outputs are frequently larger than the amount // being sent. The refund will be included in the unconfirmed incoming // siacoins balance. "unconfirmedincomingsiacoins": "789", // hastings, big int // Number of siafunds available to the wallet as of the most recent block // in the blockchain. "siafundbalance": "1", // big int // Number of siacoins, in hastings, that can be claimed from the siafunds // as of the most recent block. Because the claim balance increases every // time a file contract is created, it is possible that the balance will // increase before any claim transaction is confirmed. "siacoinclaimbalance": "9001", // hastings, big int } ``` #### /wallet/033x [POST] loads a v0.3.3.x wallet into the current wallet, harvesting all of the secret keys. All spendable addresses in the loaded wallet will become spendable from the current wallet. An error will be returned if the given `encryptionpassword` is incorrect. ###### Query String Parameters ``` // Path on disk to the v0.3.3.x wallet to be loaded. source // Encryption key of the wallet. encryptionpassword ``` ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). #### /wallet/address [GET] gets a new address from the wallet generated by the primary seed. An error will be returned if the wallet is locked. ###### JSON Response ```javascript { // Wallet address that can receive siacoins or siafunds. Addresses are 76 character long hex strings. "address": "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab" } ``` #### /wallet/addresses [GET] fetches the list of addresses from the wallet. ###### JSON Response ```javascript { // Array of wallet addresses owned by the wallet. "addresses": [ "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ] } ``` #### /wallet/backup [GET] creates a backup of the wallet settings file. Though this can easily be done manually, the settings file is often in an unknown or difficult to find location. The /wallet/backup call can spare users the trouble of needing to find their wallet file. The destination file is overwritten if it already exists. ###### Query String Parameters ``` // path to the location on disk where the backup file will be saved. destination ``` ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). #### /wallet/init [POST] initializes the wallet. After the wallet has been initialized once, it does not need to be initialized again, and future calls to /wallet/init will return an error. The encryption password is provided by the api call. If the password is blank, then the password will be set to the same as the seed. ###### Query String Parameters ``` // Password that will be used to encrypt the wallet. All subsequent calls // should use this password. If left blank, the seed that gets returned will // also be the encryption password. encryptionpassword // Name of the dictionary that should be used when encoding the seed. 'english' // is the most common choice when picking a dictionary. dictionary // Optional, default is english. ``` ###### JSON Response ```javascript { // Wallet seed used to generate addresses that the wallet is able to spend. "primaryseed": "hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello" } ``` #### /wallet/seed [POST] gives the wallet a seed to track when looking for incoming transactions. The wallet will be able to spend outputs related to addresses created by the seed. The seed is added as an auxiliary seed, and does not replace the primary seed. Only the primary seed will be used for generating new addresses. ###### Query String Parameters ``` // Key used to encrypt the new seed when it is saved to disk. encryptionpassword // Name of the dictionary that should be used when encoding the seed. 'english' // is the most common choice when picking a dictionary. dictionary // Dictionary-encoded phrase that corresponds to the seed being added to the // wallet. seed ``` ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). #### /wallet/seeds [GET] returns a list of seeds in use by the wallet. The primary seed is the only seed that gets used to generate new addresses. This call is unavailable when the wallet is locked. A seed is an encoded version of a 128 bit random seed. The output is 15 words chosen from a small dictionary as indicated by the input. The most common choice for the dictionary is going to be 'english'. The underlying seed is the same no matter what dictionary is used for the encoding. The encoding also contains a small checksum of the seed, to help catch simple mistakes when copying. The library [entropy-mnemonics](https://github.com/NebulousLabs/entropy-mnemonics) is used when encoding. ###### Query String Parameters ``` // Name of the dictionary that should be used when encoding the seed. 'english' // is the most common choice when picking a dictionary. dictionary ``` ###### JSON Response ```javascript { // Seed that is actively being used to generate new addresses for the wallet. "primaryseed": "hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello", // Number of addresses that remain in the primary seed until exhaustion has // been reached. Once exhaustion has been reached, new addresses will // continue to be generated but they will be more difficult to recover in the // event of a lost wallet file or encryption password. "addressesremaining": 2500, // Array of all seeds that the wallet references when scanning the blockchain // for outputs. The wallet is able to spend any output generated by any of // the seeds, however only the primary seed is being used to generate new // addresses. "allseeds": [ "hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello world hello", "foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo", ] } ``` #### /wallet/siacoins [POST] Function: Send siacoins to an address. The outputs are arbitrarily selected from addresses in the wallet. ###### Query String Parameters ``` // Number of hastings being sent. A hasting is the smallest unit in Sia. There // are 10^24 hastings in a siacoin. amount // hastings // Address that is receiving the coins. destination // address ``` ###### JSON Response ```javascript { // Array of IDs of the transactions that were created when sending the coins. // The last transaction contains the output headed to the 'destination'. // Transaction IDs are 64 character long hex strings. transactionids [ "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ] } ``` #### /wallet/siafunds [POST] sends siafunds to an address. The outputs are arbitrarily selected from addresses in the wallet. Any siacoins available in the siafunds being sent (as well as the siacoins available in any siafunds that end up in a refund address) will become available to the wallet as siacoins after 144 confirmations. To access all of the siacoins in the siacoin claim balance, send all of the siafunds to an address in your control (this will give you all the siacoins, while still letting you control the siafunds). ###### Query String Parameters ``` // Number of siafunds being sent. amount // siafunds // Address that is receiving the funds. destination // address ``` ###### JSON Response ```javascript { // Array of IDs of the transactions that were created when sending the coins. // The last transaction contains the output headed to the 'destination'. // Transaction IDs are 64 character long hex strings. "transactionids": [ "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ] } ``` #### /wallet/siagkey [POST] Function: Load a key into the wallet that was generated by siag. Most siafunds are currently in addresses created by siag. ###### Query String Parameters ``` // Key that is used to encrypt the siag key when it is imported to the wallet. encryptionpassword // List of filepaths that point to the keyfiles that make up the siag key. // There should be at least one keyfile per required signature. The filenames // need to be commna separated (no spaces), which means filepaths that contain // a comma are not allowed. keyfiles ``` ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). #### /wallet/lock [POST] locks the wallet, wiping all secret keys. After being locked, the keys are encrypted. Queries for the seed, to send siafunds, and related queries become unavailable. Queries concerning transaction history and balance are still available. ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). #### /wallet/transaction/___:id___ [GET] gets the transaction associated with a specific transaction id. ###### Path Parameters ``` // ID of the transaction being requested. :id ``` ###### JSON Response ```javascript { "transaction": { // Raw transaction. The rest of the fields in the resposne are determined // from this raw transaction. It is left undocumented here as the processed // transaction (the rest of the fields in this object) are usually what is // desired. "transaction": { // See types.Transaction in https://github.com/NebulousLabs/Sia/blob/master/types/transactions.go }, // ID of the transaction from which the wallet transaction was derived. "transactionid": "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef", // Block height at which the transaction was confirmed. If the transaction // is unconfirmed the height will be the max value of an unsigned 64-bit // integer. "confirmationheight": 50000, // Time, in unix time, at which a transaction was confirmed. If the // transaction is unconfirmed the timestamp will be the max value of an // unsigned 64-bit integer. "confirmationtimestamp": 1257894000, // Array of processed inputs detailing the inputs to the transaction. "inputs": [ { // Type of fund represented by the input. Possible values are // 'siacoin input' and 'siafund input'. "fundtype": "siacoin input", // true if the address is owned by the wallet. "walletaddress": false, // Address that is affected. For inputs (outgoing money), the related // address is usually not important because the wallet arbitrarily // selects which addresses will fund a transaction. "relatedaddress": "1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789ab", // Amount of funds that have been moved in the input. "value": "1234", // hastings or siafunds, depending on fundtype, big int } ], // Array of processed outputs detailing the outputs of the transaction. // Outputs related to file contracts are excluded. "outputs": [ { // Type of fund is represented by the output. Possible values are // 'siacoin output', 'siafund output', 'claim output', and 'miner // payout'. Siacoin outputs and claim outputs both relate to siacoins. // Siafund outputs relate to siafunds. Miner payouts point to siacoins // that have been spent on a miner payout. Because the destination of // the miner payout is determined by the block and not the transaction, // the data 'maturityheight', 'walletaddress', and 'relatedaddress' are // left blank. "fundtype": "siacoin output", // Block height the output becomes available to be spent. Siacoin // outputs and siafund outputs mature immediately - their maturity // height will always be the confirmation height of the transaction. // Claim outputs cannot be spent until they have had 144 confirmations, // thus the maturity height of a claim output will always be 144 larger // than the confirmation height of the transaction. "maturityheight": 50000, // true if the address is owned by the wallet. "walletaddress": false, // Address that is affected. For outputs (incoming money), the related // address field can be used to determine who has sent money to the // wallet. "relatedaddress": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", // Amount of funds that have been moved in the output. "value": "1234", // hastings or siafunds, depending on fundtype, big int } ] } } ``` #### /wallet/transactions [GET] returns a list of transactions related to the wallet. ###### Query String Parameters ``` // Height of the block where transaction history should begin. startheight // block height // Height of of the block where the transaction history should end. If // 'endheight' is greater than the current height, all transactions up to and // including the most recent block will be provided. endheight // block height ``` ###### JSON Response ```javascript { // All of the confirmed transactions appearing between height 'startheight' // and height 'endheight' (inclusive). "confirmedtransactions": [ { // See the documentation for '/wallet/transaction/:id' for more information. } ], // All of the unconfirmed transactions. "unconfirmedtransactions": [ { // See the documentation for '/wallet/transaction/:id' for more information. } ] } ``` #### /wallet/transactions/___:addr___ [GET] returns all of the transactions related to a specific address. ###### Path Parameters ``` // Unlock hash (i.e. wallet address) whose transactions are being requested. :addr ``` ###### JSON Response ```javascript { // Array of processed transactions that relate to the supplied address. "transactions": [ { // See the documentation for '/wallet/transaction/:id' for more information. } ] } ``` #### /wallet/unlock [POST] unlocks the wallet. The wallet is capable of knowing whether the correct password was provided. ###### Query String Parameters ``` // Password that gets used to decrypt the file. Most frequently, the encryption // password is the same as the primary wallet seed. encryptionpassword string ``` ###### Response standard success or error response. See [API.md#standard-responses](/doc/API.md#standard-responses). Sia-1.0.4/doc/assets/000077500000000000000000000000001302033070500142705ustar00rootroot00000000000000Sia-1.0.4/doc/assets/codereview.png000066400000000000000000001153721302033070500171430ustar00rootroot00000000000000PNG  IHDRmc=,iCCPICC ProfileXy 8U>Yf2$~}箽ڟ{u/-Aadkm^G'g^8 'hYZy4h#Ė-t@vuD=!00?2,d / ;s `m[k"PopFP mAAj"<`oCtvoyfM/eu}CCHQK`@1v!ʇlh32oe&[ AnG_m-<nh[=T3 xtM%9N7%"(s0#؍l>*"(dO?g/_}##cko`jۘ;cGnY ^d}50_;i{,Vk=CMp{x#ѥmoRHo}g<÷B#lClg~$cRH74 xA8R@0s5sȧ}@d V/wuR!3Aޞw[\=L=f3y䶕Z}b 8bMwn?:h!ZG@-x@|S@0/mO~˩Ũ~p2:mE?y[N#)^ [o$LnGŸѬ#v|HWHJIy:!Qd_o0^-D5 r++-s|>!d$V,9*Lٛl*ܱv'G[ dgnDd"P@ ` d}@ :AHgA6 A1(7P#<`Fbc |` A@8$ )C{!= W ¡P.BWr6T=B}+4 }VQ0 ňB P(- :F%ΠrQEj#3 j f` Xց-`g &ñp*p=/qx^Ac h^h;;:} }]F_ߡѿ0D 'F18b1$Lsӎ)e c`>vñq8  K]5qSxJ<^w9F|?~FAK!HJaAAENQBQOC1EF# ?qB.NxCAIIRҊҗ22w+TTbT:T.TTgQ5SA$Dgb J#f6>FG]MOFFM4M]9Z Z!ZZm,mm0" ] )tOfqBz 0?;CC C;#Qшя1&c7<=<=a5.>?-;pp_xHR!Cه&P(thm#U+\$Dy?##;t+J,*%j:Z?#-G?.F+J,rXⱩ8ǟKǟ_HpHOJK8ap":4|Rda2:7;E.BʯT4鴜S:O˜=yLwbسAg242ѝ>7iYś}0i|Ny㹦.~`v^U>g~JrGA%K\i}/bpH([QľRҴҍkAˬʕ˯s^O@UWp{SfmD*檴[V]o1rWn={VCQ5>5N}uu-jH>אaz#1q)i9y-[[ڬںMڟ<ڡDISէuʝ5Uw)t~bwuROmJo}ߞ~G/t_<0x6h>7d7rex˙WD{y:J;39VVmոwۼ=>i2tr}*C4tLìl}>|ZKL9ȗ{_5v;O}#~ۏk -cKKk˩?~(t:NEs7D7zB" HEyyD'<@.0v`IBPmpƱY)xTiju4st Ōb,QM4%\?x&=<-Lʉy),%"m%'[!7RQگRnQ]yFm7:x]E=wcƐy=˗V?mm,N;V:=s~oeyAJ IkyX{&/%XR&C 7FEf v:btT=F)VJqxİ'N$MiNJ:.f,[\ZAg}{ޱ܂KM.\)/-Y(1k_wuNe_շtw Q]k} V5778+|M^/ R +Eg-%9##,XT\:^ 3t[:\Fzv&Afc-,OX%Y'ۤڞKOtrquݧᢿ@l[Nv^G|}|z҃Cȇ\Țr5-=qzG⧗:< y.эӛoBo@ojm8e⫓#1茲΍ս=9N=ɂC4fg}19Kאo?"} ilyj:6zp:Z݃JagqW>R+NBH5, 5+frL&ofkuVQ6&u~FJb<]I$!=a^"]b↻$PROeɻ*(b{UT^yUǪHҡ? y ZZ,,Y-?Z5Zgتo:u4vbrz\/W?<wPGwwA1y됷Oo?o8j>P ,s#DE:059zHфXإcqǽuYOP&SRONɟƝ9s+=9s3eeEgkhO7YvIP"blte4H$pyfՋ[wD:;w֩.~PѠɭأK-o6u<~znў}^2lnu[wi3QͿ/Xmp[dHid yXXUeP I Whv5: ݎ0#u6Gs]}k3((2)ȚS:RC2a2Qx04M7>m3^&zNQ$3]eJgc~r͊{Ss+ۘg.ߙ] H%d2+3r )u*VFGZjF&YGQ@WEO^_Qys =KN˲9ome١¾8{yd{Yc ~-b0mYt1B1&{'&v&%cSNO%Ho͈t9vA-O@P qID)ǵnL7F+oY/SsYMn3mC.oC/'Ff,& gs放(]X^K\oX(dn +b@G 2 5CQ(]T5Fp" oz:F ƊbcGq{qEx<>?@BQ@@T-ihi]hK?gb$21i3abff=Ʈ!ȹ5]s7t4?V`EW"b5%\% ke^gW0RWjURuQC6je֫4p633hlig-k3blhb )mC3ߛ8,Ԑø[#ã<|-:>:O=)2fwjLYɌWj9r/'\+THnige@J"wߏ^Mz0Iy%1g]ݗz_btmIx*~5ܗ/_ CEK>KK˳?]~TWCVr׾mnrucڴۼ^rD flssldlnmnn#vپkh:UY͈NiTXtXML:com.adobe.xmp 670 192 p0@IDATx \Uי/Wl50c k~! Azm!/8@ ^k$&3[E +$N83(8A˜}Ƥk)>zֳd B@!P( یL_W( B@!PHA!P( B ;D!P( B@J B@!P(#0J B@!P(B|Ap\z###UB@!P( 8{1{접:%at IBASQ B@!P(J9f]֫ B@!P( @Q؎1Z^:O!P( B`"SP( B@!(vh* B@!PA@c Q B@!P(exTMB@!P( 1(s $*B!P( Bv "ES!P(wׯCU}@ $$f͒(~P' B@!0>}aبp"  .] 瞻Q٤xel+;k \GN``p #[L6 0Ә80#ԉ΅ӧ!d̍({Ms&HE@!P(2:?. Ν˗/+ee71}3h@N6Ӧ"5h OF{x&nd"،|jر}-7NT( ?]K&"lٍ7pyuv$-<g?3gNeuҴ<%lnܸ0K(͏@!40M}) ihzMft_IwjB@!P(>@QQQ?>eïk`ˈNW:bbbH}w)7<(ZOIo&i@Πg? %{fΛ"}:~0>,,QZq}; a掸W|k[W#+%vL?l=z<(>|U EJJ_89,ig|ч8y7~Î<&5̘JU(7?snrll:uJ~?Y^x)7S>6齜27G?IS2d:f⾹!?:sg`^4f> k~n⛷Sx7 ^"]0wxgPNkuPk=(@ewyN ヹv_ML`ָ8n`vŅpnj;iǠ)./߄KE~L7>x/TS7 }^~N [0I8wI׿3 *BO bOG#zNMlPqh{sW| |'[OK,it ۅ)e3}E9 '&[Drs6or\i.q`8KG%Pss7o^G#ÈRm :D8/~4>g-BHrL]Y'}myt[ΊÃM^rLJ\~ < '?Z OqB)^[ޮ37_;61l;f.~[n[G4G+M[7ho&rtT"KHL]Zm'1퐓8U{S,r Ͱl8Gx>PQv0S3 .t^?wĞՓ'Ozq"M乓a^g`dPtF%yH, `t?D>{)xj$7Fa=|fgE?Oݰw=^=ދ}/#Z_1^O Ddl؀/ꯀϧCڍWu>-d̞Y 2*~NY>E8NtOqy">=<ዧPj5wP&>omb> # ,8PTٱ!=7z_=~w/ןt='b۽2Q qF WWӚ ՙbtMTejtVN'4]s1 0p8Su{q.q.knLz'=. ǂ`Iau{MYc<5Bc΀ sds"9< >1 X4_~;OJ"{+^翅gݨm݋}-NP P]>ن %+soK*Y^#xiBNAEh EE+6"뱇0!y{Ӈ$ܲmɽx5|#1r=쭮]RC"dzyٌ^Wϕi揑^8HB b,G7 ]Z yI4.\AƷ@l}~P9o(g+oDC Xu[oδ lژ]baoٚR__kEș_bgAhӽF[Xz/ m(zs[S~3up\- G\zsȇo2\Ɖ2gVoP1KVS<"Yqin i<} f$NМEˑj@]!X9yd1hQÃ8G> 6RK@FeIhElT5#&yڎĂxzǣ%t o[4w+ʈ?mr1/ϛ7UDž"qmr祋W0fY"lfI =aPQgM\%ea1yԂ?^qpog.>[dceDž罛{ p8~bҨr,:Iywg7+6WNNIlƫMre:p]ͨ*><]3 |} ONft>g ϼ#W~޳8Ǩ>)"0xe}wF,\DSu~~`}ξq|rK\+N 빕`# tD%obA X~5uJd!^yFg4V_9}6S?)EWck{UiD'賝vuu ( \<11qzKSput0rWb~e#x՝4:b~fJs)?aay}*Cw/h51gFl\J:=6&_ ֆ92y>.l։"!70mǷa.tt82A-`t'p[~N .rVI`x8Uo" X('_Á#l4:$gr:|l=pehwOk/XK#f2>|}FgLjֱ PB.A`]9v%㋿ itΗzm).$v4/_s'Qjך~;'sO~s1`փ{9{Kxk_w^;Q(BV"|^8PSni,Fet@Qd^>Rw-_N#v/>spKP'Zi_sI';itPo~'M 8N3&a)uct kť  6_DQv) ϧ~N6O0,ix>NNt*gq!4l:\f!rn,=\ìW+>[ DYGN򞇑G`+;B\ׅaF,_X5҈VF0k ͦ'FKr/W:A.s?i}`#G<0loAk>Uy^8{Ϟ)i޸| ҷO _6žcQ  Extk86q韣wv'~_l=ٟ﷋Pe.7;IƊK4Oj7\v>ј9uo? hZ xaǜHrC?⨀{SDЃŌr_k30|d#2m1җǜd*Ex޷7" &v>0)ޚq8 4* >x۩蓭TXt1">UkQց_蜥>3k\x0W<ǧ|8"pm/"VCݟ~IeRU rAGg tP`R?1KH|z=XL3\މ'p#S'BC"kǂS}7ʒ+dRg"N{˂#q`]IǣXl1'&7ЦuJVs>O} 6Jah4PKʣ+>DlO*a*7`o>멻bCNB$+AaA:|zr%apH 7~EC,s&"E H3ND#W(cue oɌ[0> S0yMx&Sh" ޿w<6DbF/y]?aw>&].#^1]_DSE-̆'뿜%R 2ݍĥJs{]]!1itLdqJo{yӓNk5K#8}G`<=HhE9W?7Q)D+umoYYsh0\t!q~''1VL"!b͌tl,f6tINmi?ҵ(LZ-|^$D4Ǜ 8Xr+91AFzD'KqY:ƈ!(]QHKō;@%POMD \,L]s >;jwua&R#=BE 62kMVE0Ҏ?-V}*>v衟ՅG:dcG^R1N!r|G?D6Q31n" 3"s9Ns/*3.sYpƭP4:ECVqCڍH9*2 \tc A^2Čiϫn't}4c>1|d{*,-FϯCsgr OxWGrɁ_}! i@.{ }F"PWnfb|BgV IX^Ahi'c< Gw2XC~{\gu2F04ZA ?gKϠ3G<Ԃqg?mW: "8.ʈɄ;mt|'ݝô>B/?9o^iX3s{Qo?y>%\wipp̦& ߠmzWFAAWb׹Itk =F'G.Ǎ$`X:(c(Ywa;?>N7#FXIyh Oj+*Gw\Fh(֤#:n\.oً}}<(B[,x?*(t1#1^x{RέHX$ N eqm\"Oա41qt|,7\Qth8XǛqy,Rj'P쾃`N9 {e{^:|mOQ7>hc&ai"w*&b^|/b#X!d ^,>9y po)8n\/yޭ5'/9h3=S 4ꏢ2n;{x">Eo^FfX?警/Lܿ:_4:\3X${V MޠXa=Rep)s@ +xvz-cp& {y(۴qߠYp9;|ϽvfZ3ڷ9:zGQ{9<| SzY<ډ3m ըov:M~Z'1ܚ-C'o] %'!ꟻ)̯0ǣemt C{ϧW-*#b*,;ãNpy|4&??7o!xH*#L|:G}P40_劶8=E/ žγяȃ=8{,yh8 Z?܃<{jiG,u^:UEqy{aNt|/5qz_]ir [ߊGof/ a_oنkH sOۏi;NC\r 7%aE[_3n] &TM4Bk๕vLŴSKB tMz\yw~_|y85?en3x _C|W܋Y.w<.U܏wKz|n#a&Pasޗ1/Eq1H1cxAdy|QF=Le|C+zGO6g>3\jO)ؚ4? ]T0.>fraN5\yag͵q]Xaޭ s!RZ7Fgr)Uk j95n\E:jE\s#ǔĭBeZ0$GH>ָ DEbo':vWytL8t'%HOCQXs`\X%&itάE<t/ 2g[^Bc' # r ˺z&b \*\9ٸJ> X)Q ^x0r[1pfV ^6DjW)l=MOfӨXFdIOO>~OϢx(b PZхH 9]/ϏnW #zLo PiYv#4#}'O.l|7 ;i-mqLrZ$<;=1N&< /oݎy?3n\Bghp0_O7G$l!}2mƼIx pĆUA!7"FJ)n)ŏB@!aH ۿIoh9A"goi@"|N٠ |Le% nҟ9%Y`tc s Vi>:}N柆auLN0TP(>B>5w8R(B@,f$)EDbOLq kA,%S BBx:*or&0\W4Cz1^ݳ Wb឴0NJkx8P.GLS6EO!P(>9ĈeԻ0'SWw>Fkay{7 ;8L _idSBXL+Ub}a{0|¦Egd!r$۝454п\-[{zn B@!E7v aبp"  iTq .rC*N!P( BӉD Oul)U B@!P(:yubH!P( ex~:UJ!P( ]2<.Q ) B@!t" OgV) B@!P]%!B@!P(NW*B@!P(w>HlWB@!P( T"?/B@!P( B`<OR{H| B@!P(B@OV( B@!exʧP( B@!0)9)TaB@!P( `PgH| B@!P(B@OV( B@!exʧP( B@!0)9)TaB@!P( `PgH| B@!P(B@OV( B@!exʧP( B@!0)9)TaB@!P( `PgH| B@!P(B@OV( B@!exʧP( B@!0)9)TaB@!P( `PgH| B@!P(B@OV( B@!exʧP( B@!0)n@WG;ۻz<{:`w IM3;GOڻzђwGY0=5i$E]~eh?eĸ@wMs&)3?R/;Q揚YfBm,:)>77<= NVKC7)7<غ)yyPBkSDfi&dVwDޜMu]~,ev?@Ofmh/' ^wې]P쇐)nCmi2TCЛ ܶ,m؄nbeێDBLgQSX:ƁDOA^?C=M z+jڠ(n5 atvK*w}<|qC[o/deOcLw&?fB͓[3"!A@0DW;I鰘J:#L^E"mDZM]!V$Y=ww53|gHnswm17bGOm#ln]i+!6V-MJ\L{{nG1mJq&2_OyEwlM8"!W;r1cd+u}>w1Ӻ#bžW *(LmHM(Mc##oQ7l@Z>PM[,49R{D1*%b[(Llk le=_Ѫ0}h7Z{{doġm./P.˕ޣ=inB5v{ ,:4HGF+ [P_lw/zZ*]0ӌ\zFe7@v^Kuckv65y-[+eOn!W` v4/ڒN<%/ZFZӫa[F~yؔ+=CTWkK3£[VՈtnG]!n~G[5/4v]`OOG/Mō ]ȖNl{`ko!rQ&Glu$ιee;Ux5[qe  A^M(QI0pF4צhWގ\y!) ] ͞.b"eZ*P[>|MFOG2w]Ue㮲äy NP[@e\!=f[<.fkWuro~moTùX׉ccz<1wa;TۭBf'/t3bË3A.y$ǩ@][oH6TWr0䭠 =~:yǬ'1ni9P}y#y_vT੓>1vǓ9gHp#*ǘ.>p[ O!)EH4uPW3ZT֊rY T5!)׺[a1BaT'|{)2lؑWL%񚷵? i)ߚVScGNy9JszOiJͨ–"0Duddkޫ$=#;m#_fl('TЫ'I+aPO#P&ڊ%husxa㯭5U'Bz jd~kmKBͻgаٌšHW*ep6V!rakAldɞmL"E (n3+l2DlvF h%N`ĥI+)ɾbzw?eK7YnȁVzMP/<آ]t bqaf&ZS~ٮ^>.J`72qLTw#2emvf}PR<87gzCVQwi~vͦՃmO؛SE)*lxcRRp?ٶ *nYeT4?>jV[[aurXC7skS+69(YIά}Nwi,Sk= v2/DN;)|L?rFsjc0)͗4:%I󮒃Ά|UKNgLh(b>'im'j#,g3:EΈg95NWNW5ވs$G,!$Y{D/.)A;sX=#/mi|YcǨ^md|+I`}gߧcͻ&^IZ"M$.Y2 :rIi(%N֬GN+?&q7ryGjFuZ;HC֋#FƹȫyghptD]J{n#_i܃yHSL }E[1Fpƹ+rɣY61.ki0^}_$gQ7!9.j`= @cȣZ1}۪Օ5h#1$&] +ͥߍvR?E G$l51])uqRRSMKA9'%ieO^(?Xys;Զx2#$z⦠/t^Lh>9zx`Syi/iƼ.ep~2x޹%YNKXs[Gh{˗yT }* ܺO_>g:\:' FWǤ֭m{D9ѻ_~"c$>T^U]]dJA\[>귣)[R-  !5Q.:˨HCHϐ3d*]2GԳHv',LE;`z>FѲ^JOhzX.(<ĥuax ȡ; GpKY&#ߣUw땼 Am&;)vG ;pS+Ȼ5> 6qe&%JGoϦ-6|Tl#v_9%-V4JuhݏzW:S_X ' y/^9ŭE.H7j =\R.!tq@;^QWo"  - n"rDq8q0 kq̚-QWP *G#Z go!e)Zˠi&% z 2dlj0$ǩ>zHb~G00yY}&-5ߒA^39;?XeŰ`3"(i9k A u1 %R\ǑmGvp|8ԎCn>9XGLa.$§Z]Z&pg_ZYṫ?}1<`Uo#۪7x0s ^螟]\c]rAov]É"!ZbEus cېĭGu97ULXI K-C[LY残R],CvA·;LhvӼQ`~u,q|Rq ˧^Feζ7w )nP~ЕiUga9Iڤ`M0zyaL&rg(GsA%jUb6GII9-\G!VN溷 }[sTC\2/1':4\ϵ@"<M.?FD#QLo嶊XWLJJ$,QI;PǸe6r $e˨ŚDȟ‰܂(,ft=ŭ?5B v - `hǙ& Q, R%>롱8#n>t׷I/zq{q aM-x],%zv${?z}u4 Uեۼd-Vf4գBWϿWX(Rק~e&`=I `"TxVbk,:p2(1 rMMpG˫q0Sae ď$w&֖Ssفm&܋=M;`88[x=sG:ݨu yBpvӈK@|i 댱$s0 aì|IPVFh-֒VL1PA^Mv0< w/LT\8s:IY^װ9!`nw Oy=)8uF)QY'.~ey㖭~߽gSfg<F#tiψo]_JC˯+e>qF{aш7&~ɽf=49tp(c18;|Ý[}4q)E[5 JGj qr1jy+_A{^!;QɴTIB3F}wv\( F[D֛[xKTtWhxjrpMߒ5[$'FeVheL[$C<Er8O{`g7 ŷ){i!"ʦ]<!K-C>| SX("R'#.䏱I&t&Owp+7g4ON`'<@IDATDFNAOL .X?v4rHi$s=j+v2[[=2zLm5:wHqvػxM҈k'xw]bE!|CA#]`B`XƓ &8Dgj6DTҋ5~[< KU⵻^:e_a Xy$F l8޸MGF>3'݅yXen.Mylr64s{]\R>e<GYt{I^S]]< ;f1H'>E}Qȇ5 hq0 ZKP9㨒 ,E65s~6` z`[xWLvwpȊ\nBS|sٕRgKxOAvq#-Vy?\# KwĈ~ݢyAQ:xCwZ7}IB$y| HNRt)=GH?i0?\]~CsxEPȤ%W|Gzo0ު~.V`dr-n|r0k-02=,WO.,'ŵ'0R>SU4Pzr2W*~rS,?U\Vlރ=VM9%FD:&Gĺ=piq+$Uwj>xsO29,-pfV~gn^,8|%T^+D xTGx#i^9[.gxO͗"$Go (-=yuɞF76Z۴ +}o*}rl+H)G!OnPǖ]5DOs6 E(5NMݛbsF?#g-Z9ajr9BL*/F Vז](qx%(C|КF!_DT7vXfQU|3&`ۺ{yenkmZU MĤ]8B1'ĺ|dד_F9XӾ4ȳ䯦G':gȊe!䐆 [%̱\S#L|g=࡫x^f֯-3'u\mȹ/Z6Ws Z{Jv֮XӧqW_{=|S,·Ej4oaO&3u@zCR!%4mwlވ]~D q=x{GĜRQ)k60ӀGQK2%ޖ)JT#l[c7ֵvO)oo.&^ٵN}R>b+S/KQP \@1]wC,:'; F6likע0#$C/1VQg>4Ie'}ϓb {9'Uy]r6t-.ΫWDtߥƏ%]Ҝz/T|[sߑ6twy0}r֝2+QyucDNw:=XzƎs5N0JS5O[fJy]X "o#[&La<(T~)m}jDtOs^Z`Psl=)F_>wX/z}# ^?udpP` bVH@ VN|A5GFyI>tl|(AK`yXݲa Igvd_D4Oez{g!|0WqY|Vbd?*7o%%'S^' s"S,397)?qaC|Vno;Bέt8%NNIf  B@!P(>8'P( B@!b9ŀ*r B@!P(Po\TB@!P( # )TS( B@!2<}b B@!P(exN1B@!P( oP( B@!0L@W{;:/Sxu=se Qg/ =]c6t˛1#Q}VD볷s[&.K%]ﴉ;z&'7]=Ԕ[ԯd[C}honDcs{Ri$2˩lPzRڕ.F cdL T4exs"]ܗ6%gyU%պ#u,cwΞ;'#H{GuܙUh<-p&v]-$M)@eqVӽ3CWc{ _2;TDKCce1~-~e m_fӘnl15aodl`b˾nfiyG|Kwzhv2C t50t>gRS a79]r-5&|pys&:Mmptpn\ l9iCw|>#%ԝM.Dߥu*u/bwQܔDM61qv75;Ꙛ8 2ya )գdAO/hF)zMiENi9ʋPUR*naFY_߬nEyy9rQu 2g_6dn@o|yۂX-D r$n4EHoּP'$|7ZٶL!!<QRZ6TfP5$#aV$1hAEu%oM uD既4? M; H>ڱXpJ&*껑ST-kcTUv >\yZF\; ˑ%^CX~tKI )E z[?6z`޷lh5˔!ښyAY-^(Ƨ V$qW s~SAv<PR~!7.g KjɉDNL+QRӄR(oC!ǜ5KvU1&o i?9>19s+"gO#,D-E?E~fUahB mَ-(ݲ6gff1ͪ"2!w~+PX/8C%~U|r%Vϲf#UU(ez~FC.Su1[> i&ǭ@>J~qXyWD[>s 9M魇Q՟Cr^)eoa߳/؃@2ӬVR)>d [7&8rr >)͡aw&?W^^!|D Y,'0tAƌZAB_bu!(&Q/vrVn]QM 3.)FwV9njBq`Ɯ^`|BļGl91>#w 5Uԥ[JsZ9ٍA@Φ]ܺN&uݛDj#roY[)470Ig&:Eǜ侧=mnrWw戽P,78)=2.:t}TgڟwMKr+yUV*ralH)b?4״wɟl i zIENcR}=s$kGr|l߮6b`nF,j8G;^ lpC{t<5uk2˓&#,oZϙZm{ἳTcƎ5}cPNtr^mxhR3KHה2Oq9߉ʻ L7x1c}FuzcE]ʏǵpah;OV&c1ș dP>N ,gEr~OWۏ9nݹ+qd,#Ag:cڷrnߥ u5hhz^X+ՠ |/ ] bjiy=N$ʞL;B]RֵyKc,uz\]c3[ZQ L)oqehkӨ}3ŹxgPMIuRs(%Bbɿq/Gd8 KQ+ VU!ގՔ&4r@ʖsiWH x+Dd #cJ#|b)YHp Z v`hGSM18D5ocE` [aL6 A6u3J*=M,W=eC 2P>?|$Ekg\DV1Vƭ;~'ݟ[S~+(OANFVA,ϦoGŦ&ʼUXQXfZߓxy;s|<Ȕwa8Jyy>_a\YeqFS\˙+kqf/ɔv"PZs e2EC)Zk.GymY&ӧ\&GD>8d;&$d * fk.!!*L8e*%2Sʠ:Sb̅'AyD!չ*|\Će\(LgNSgDWGG52=7%%IV32է=>uC:16,mmBp쥓FqM^ĜBJ)n%K7\& yT2`\PdU?aǾPZ70&]:#u8;y*L}e"QNġY]5e`eN҇Fz/Z?OTq̉8)29-~URN_|A6T962ܼ<\E>^.GFL7;NN]љP1hD4e7 ]¢GfxW$?Lk[>n&4رL3%/M R; n;zDKFbmTttg$H6H;;$ 38:2-+eL#u;)-\򃧣#|-eLMKm` ٙЦ@3g헑fqbFܔ,E-iGnӁՁGr~\ 7Vˮ]Oo~ER4QiiYUykWg͔v % \Mr jyĬ?+i>8v=l>PzpeZ4K0kȌg~lEN4v8Ձz ڃfu%mn;i}DY -m0iׇyJX?>jA:XkM'MV 㨴"S2a+c]/7͢' Ss ɪ>ҕ^=J[Oe7k#j _7.o;"yz-$+n{e=ىFi)1Y+VTmHrl(Uvjg.TVc|hIQor ]nTBJp]Cu P=+e|eުdMԱkmEosC8)uT%'d G"@v*ٷ*)#,oZйB/)WjCʻNNFkCk~B H>J5 ieUȠ %܇#NO_Pv5:GI DXɝ{_Wn{ ]\҇TJ/MU]_P0؁P1do<$MK6|֗e*Ψvl4i_J+( it}gL9 {c9(~uu("2tG$9I+M_q$!}뀼cX! %yӁ1bʁ\LRig cGN#UF= ݣT>n@moKnG"(ܒK7XSe}ErL> P;dϞIP@@=aj =͖_!²ZT*li0v#Y%eԗf3lD}c JmMf<?`C^s`w 7ޙ6&m?W$[Vp|jٝJ{OvΤ*RՆA#Sޒ_@ly(iTV`Wa\Hr2n$+vqeL1erPk04fޙA6ku¨q7*1itR s~Z^vߣͨ;JxFgc)B dai){TGQvBdDdz:β 1IwLѷԶdS2z=b)Oю7h3}I=F,~™oy9 8!M'oT٤&|-HGyyqI 5t6::n3RuIim,UۉjF҈aUQ^ a=W ]ե&JzTve])lʺ̚ %g^W>7!r6%|vPW IލZbY ѯ+=ٍ/Ȓqt9hHۖ!m,CLJ9C^6m!%g alC uf(͔:VʎOD8 Ҭns)ϖ4vϡ6Si+ -@tȲR '[u[DC^t|6/ԖoԦҗHHHH`f6HHHH`kܚR #@᪄ $@5땥"   GsU "   Isk+KE$@$@$@熫D$@$@$@[~|^9xOё dO`Sπo [WǔP==e    Pw(w̜HHH6+3_P{=>1xF+gB !SqCM4{DvU_?퍉9aރ/&{=,~ѓ1K`qf8փ}^j/Ķi3%1\ x'%zLD lI7@xH-Q֍13ы(]Aus/fP6/`G-'wXqա%~v G_͘XvEW5e]$(~LϠ()/}%y|+"tMq ̞"p!~383g]Fs8\u-lT?D\\ Y){qC&zg`q"IHHHn9jMk99|9ڇRG[ TL8jK#)U('px_:Ŀ5"$v.2y;*D>7].p<    -F 盋z=>ЅY޹IxDtт"'DaX.`rȣK:s    #Jt~+_ɘ86 @3`$@$@$@$@$ Ϭp10 Z Px dE3+\ L$@$@$@$Vk%x$@$@$@$@Y  Z1 @V(<$@$@$@$@k%@VrG$@$@$@$۬B?ig]۰c|k_d\   F Of^z~*6 ۶msޜ o;wj!  DHb[գOUt$@$@$@$!SbE95AէW:   XO 4skn^M$@$@$@ERxRtn-iz52K$@$@$@TXNHHHH`= X Oͭ!6q=&   KI,$@$@$@$@$k/7Gr#   u p֭{HHHn*Kixnznjbf$@$@$@$GRx0kĹf\X'~>|HYqKpAϯL 4˽x8f}ھun̻?Ʊ/~Wͦ~tD~ :/x2y,]zYXNc!|!>TWkol~;Jv)%V?0olB9".' ]0N/P: |mҢ/>ahGg$b璾+ϝ&ֿxlByW7TRk 0$_q~c݊ؖy=A$@$@$@HRxf3fL4Vqvp0H1>ķ܁;ꪬ}-k-?PmwZ'#lHYQ oѹ-O퉿soJZ-01 GFy0j4| ˚K2GlvnB?<|KS;}G5۶z[oW4ݮB%_sqÏ 7_6J_$@$@$@$n,IeieQX6^ Ѳ ʼO z]lo,- ]"(ʼnaỦGvmTNM}p-tNeӻ!'R߮yK bzwp/>8"x%W[GJ)1п޿?)MqB?jm]iǵw .?Ⰴ=0݁}E9S[:~ dFTj1½{55؝g=?_ zX:K6Ԃ[DF4DgM $'"~u +Or{T% TE,~Tz#w /d% ^M(: $  DpZ"._]w7A\[=yJ ƻ5^ wx V%g|$ojSعs1 lIsK"   XW Sj 3'   ز(Yf2$~}箽ڟ{u/-Aadkm^G'g^8 'hYZy4h#Ė-t@vuD=!00?2,d / ;s `m[k"PopFP mAAj"<`oCtvoyfM/eu}CCHQK`@1v!ʇlh32oe&[ AnG_m-<nh[=T3 xtM%9N7%"(s0#؍l>*"(dO?g/_}##cko`jۘ;cGnY ^d}50_;i{,Vk=CMp{x#ѥmoRHo}g<÷B#lClg~$cRH74 xA8R@0s5sȧ}@d V/wuR!3Aޞw[\=L=f3y䶕Z}b 8bMwn?:h!ZG@-x@|S@0/mO~˩Ũ~p2:mE?y[N#)^ [o$LnGŸѬ#v|HWHJIy:!Qd_o0^-D5 r++-s|>!d$V,9*Lٛl*ܱv'G[ dgnDd"P@ ` d}@ :AHgA6 A1(7P#<`Fbc |` A@8$ )C{!= W ¡P.BWr6T=B}+4 }VQ0 ňB P(- :F%ΠrQEj#3 j f` Xց-`g &ñp*p=/qx^Ac h^h;;:} }]F_ߡѿ0D 'F18b1$Lsӎ)e c`>vñq8  K]5qSxJ<^w9F|?~FAK!HJaAAENQBQOC1EF# ?qB.NxCAIIRҊҗ22w+TTbT:T.TTgQ5SA$Dgb J#f6>FG]MOFFM4M]9Z Z!ZZm,mm0" ] )tOfqBz 0?;CC C;#Qшя1&c7<=<=a5.>?-;pp_xHR!Cه&P(thm#U+\$Dy?##;t+J,*%j:Z?#-G?.F+J,rXⱩ8ǟKǟ_HpHOJK8ap":4|Rda2:7;E.BʯT4鴜S:O˜=yLwbسAg242ѝ>7iYś}0i|Ny㹦.~`v^U>g~JrGA%K\i}/bpH([QľRҴҍkAˬʕ˯s^O@UWp{SfmD*檴[V]o1rWn={VCQ5>5N}uu-jH>אaz#1q)i9y-[[ڬںMڟ<ڡDISէuʝ5Uw)t~bwuROmJo}ߞ~G/t_<0x6h>7d7rex˙WD{y:J;39VVmոwۼ=>i2tr}*C4tLìl}>|ZKL9ȗ{_5v;O}#~ۏk -cKKk˩?~(t:NEs7D7zB" HEyyD'<@.0v`IBPmpƱY)xTiju4st Ōb,QM4%\?x&=<-Lʉy),%"m%'[!7RQگRnQ]yFm7:x]E=wcƐy=˗V?mm,N;V:=s~oeyAJ IkyX{&/%XR&C 7FEf v:btT=F)VJqxİ'N$MiNJ:.f,[\ZAg}{ޱ܂KM.\)/-Y(1k_wuNe_շtw Q]k} V5778+|M^/ R +Eg-%9##,XT\:^ 3t[:\Fzv&Afc-,OX%Y'ۤڞKOtrquݧᢿ@l[Nv^G|}|z҃Cȇ\Țr5-=qzG⧗:< y.эӛoBo@ojm8e⫓#1茲΍ս=9N=ɂC4fg}19Kאo?"} ilyj:6zp:Z݃JagqW>R+NBH5, 5+frL&ofkuVQ6&u~FJb<]I$!=a^"]b↻$PROeɻ*(b{UT^yUǪHҡ? y ZZ,,Y-?Z5Zgتo:u4vbrz\/W?<wPGwwA1y됷Oo?o8j>P ,s#DE:059zHфXإcqǽuYOP&SRONɟƝ9s+=9s3eeEgkhO7YvIP"blte4H$pyfՋ[wD:;w֩.~PѠɭأK-o6u<~znў}^2lnu[wi3QͿ/Xmp[dHid yXXUeP I Whv5: ݎ0#u6Gs]}k3((2)ȚS:RC2a2Qx04M7>m3^&zNQ$3]eJgc~r͊{Ss+ۘg.ߙ] H%d2+3r )u*VFGZjF&YGQ@WEO^_Qys =KN˲9ome١¾8{yd{Yc ~-b0mYt1B1&{'&v&%cSNO%Ho͈t9vA-O@P qID)ǵnL7F+oY/SsYMn3mC.oC/'Ff,& gs放(]X^K\oX(dn +b@G 2 5CQ(]T5Fp" oz:F ƊbcGq{qEx<>?@BQ@@T-ihi]hK?gb$21i3abff=Ʈ!ȹ5]s7t4?V`EW"b5%\% ke^gW0RWjURuQC6je֫4p633hlig-k3blhb )mC3ߛ8,Ԑø[#ã<|-:>:O=)2fwjLYɌWj9r/'\+THnige@J"wߏ^Mz0Iy%1g]ݗz_btmIx*~5ܗ/_ CEK>KK˳?]~TWCVr׾mnrucڴۼ^rD flssldlnmnn#vپkh:UY͈NiTXtXML:com.adobe.xmp 1052 389 bT@IDATx\]Օ6 @CR!L$hbV2ъN:S⌵q86m[)N}Ig|_֠;I:I#b I! H.\=Mnuw8k?ggIp:E@PE@PE@PE@P@r8)#E@PE@PE@PE@P,TUAPE@PE@PE@P*2TE@PE@PE@PU8hPE@PE@PE@P# C E@PE@PE@PE@ZE@PE@PE@PE !*CE@PE@PE@PE@HUE@ ?qE'Kl m1ֻ"("("7D r0|~?q(㹠V0ʚӧOϖ_¥! Qx_B[t0"("( .F/1c?%NsCC[x-"L蘩P0K{HbإjU8Lo)2rg ўc>tv $2o@!Ch -"(TSc*F~2an).p:7B? >` u+~_)֭;hN"D`t̶_03,XEaI'% ٗO~\bN16܋}g%_ZlAfl? 擳b|Ɖ=(Z)djGO[ O8+B`JÙ xV^hN<иaնumL+DK9']8K4`l]NLB$(olmD0YaS KWVPFZv`_݇!1(\318#]Arvo߉$ʟBd"(#=hݻG"5iB,Y4% qv0 `rJDZpM2ޓ`VF cV9Y&#wR\>op;BxP(?λL)rVd#/%t-6]$O8t~"8hX˦9E#/[.8 |?"8).ϒϣtfDr >v^" %/vw#e'x x /q)aK9g.!I %؃'|@ɵjG@4"%M;!/5GSG 2~T_9oE=V 2\]~ Ǟ-&oFaG#ͩ  Ch81;j /à#95Lc/< c-ɣQ4v +A#;0:MZ^p-֯[lؿs|b?j/[u w3Hsf. sK7fun& c!ñיΗ1x>XŞxr/+A17շ90vQX <4穨IeGzd%6c=JEk'ґ63i:p0LJ +x\:wσهB>'f ẂFR#DqJ}VdiC#H!}Wa.ip?BLc;1T7?A6egcՏp~_]ƅv[6pAn&obGYƷ^?Yr̀gאATŐ tȶ[u[Pb#k '-*=܇ I}#\6$ Rpiܿm 3? ٢#=iΤuRDp"CχYŎAU8ndf1YE(/ % 3 KOMl0"/KVB> ˘k*9қQ"K+BQv +Axe*UkW}r_($WTYBd ?d%L-S|xqW+O\zs^`;?I+LܐKҊ %N[y8;IծP e`x,D6 e2e;ml^w2L. wOݟk7lq5sa \b[Nƨ4^1T%ʥl\ ~ $oIq) l,J{[ò" ,./2edH`ƛ# B雳>y7;i3cl W/P6HlP}'*,fR aAdrm:86s+.crx[]֗q 62G)A嗿?߈KܿT\aRQ9!=uB^N6ʾ엮L\v ow)(! Zv\d |˘}IsIm^)T0B5tüoc^f0 JIGx1o!y0F?k󝎾{q3Uun] 38fK=3 ԃOB>-wb[slYW%=U\|cX S)_<\].eİ ̺[Ƞ xry>ԯZXp`y9L!a .|vCAG'Ob+xnlTǑ>x&Gt*t=~mgEV?rk761ڎ;MY\?'CYژM'拸oͥ);ߊwq@Q 뺱!%N+xskV7L>H37 ߭{edZ]oKc `^z!o7Q~74塏on].k0&4}"oP}TƬ;I5tՓDgo(Vɋ þ;8Ida:ٜD?'$)+luL2o l~v|u:qo%rֺf8_[|ngV*DD>dsexXHl]FѶI6?ڀnK9VJ>K.nA,q׾%Klg` -(CaPEu+-SVeԄXbc_B_XE} k .B՗L$O.(XŹy |`~ӏ,,:fPY(iބK#3)3۠p||{a6R{ӋG^_j>!Zn9f]Y뾁m!9BgV´,d\d<PFi8pR\a_ ?pNe8gia=wXO;aɷwK+x܂r_!=^yͷ/9uj 4}1 ȴdP 킫k ͢vq)_5_\'^E/UF t^ >bO04eE1NeueEM딑y>SW޲hqK%.,݁޿yYͼ,+Ŵi)&!niڎnʷ!'J8K9Nny%F lC,&yEvTb]4d[Ȋ;nَr34'Ï.tUɫ6^䣭~Njs^ F~E\Ɋk8:0AkkU3 $Ľ{^㪵7CdO12I+N/mӮBaq{  , "ĉ)2eFc/kV}tќc˖M$v^˖^6O/8pvrJ(+*xI ];| lx~K`(TmL}6uɼsPQ^bn k6SpcK]H8r7sFSC xi?\.y0 fO%dQ9HNMO=t>0r#γimfSb5jVgcb_=N|9e|U;Tؒ 9PQ-1Ĝ Yp SJV#<}o& 'ѷo#W 쏩X3WۇwI=i_2}Jp6BCzG玿E׷nD\YUA~.gpQ'g.I +ذ3zͶ*RZš{w1vsrMghy KGXNy f9?Iu4gameWp{WE ٱ`gpB ˑQ#=R47=\~eFnX4иd|/&̫1-< o.FD\Ω z>R? J= e%VܒpbRdZxMKiQ=|_A8. [O ox#2!@ɦKn?$N#EA}%8"[XRk`pO㰕<$WHc.9Fd"X!F3ٸ(.lNy;~л㷖PW:VRtf22ql ^dn13q,,Iio J@8Hn+kPKEJkGqrr<|Fa{؉%yS_G}",y,"߲<01G1W"M4W+Z,>v滒w2 UeaGޣ*Q'w5MʳJEbE5j[D}h)$} 2F0e͏AaIм9wP mvntC[."SM$#l+SG5T Ńl7g :.omeD]2gÖK0Y!HsZA0SrfpzW2{ӟ1c茆0;$XgU˔}Y(:p% 9 EeerQ`#K-EpDپKV2=gOq4258C"[!=࢟c]aJ=lQd-.y.b?~;$ F#S9 3s*ƍ+Op)g0aA*J 70Xgfӝr4梂yCgfт#,ȳ;C͌O˸b1/Ex3^d,=QR0QdzVDw8ndYO.7 (J ׄ `z~۟lv9pgFtMNu]vކŎ+Mul?'0= ;L}[wP}qr߱p8{oO\>edAu&pI0яpAI>Y ; U6Hvr.qwi>Af{mґlw{q0*nwt#(}84OyqkNۋYsZ ^yT|ժkvͤW#-iIӢw%|qǒ萔RSg+*D'R| C2ZZd]Axec 7gxmI^ .ajz7a8qjվDY(yfKK^7m1z `@;%Ay%h8:$lhsDp!6dn-XQMV.Wy[QíHٯ\ODnj&b5a ߋl+PE/徏xt;)]0oG |*5Ɖf$"da~Ԇ`>.΂cő͐1"nosTso24p$3_:>wL'cnXзEx0W=◰on<6Ẇb |a} Xl'xn.. 㝎X\GN"--'!䡩Xz(dw6d:Si{:!Ă'6ࣳө0OVW۬hws)e/{qYfW3=; `^=9򸝠/xĭ7q?:yB%#{e)Ag[o7keZ! K棦60l%q,#17rŠB9eIJ_DX)2b}kgE~YM&q[Rny85A?k ^E+&u"́eK]bj"ҲAgtV%ƚs /W]$қx__ [i}btl-Xm_.ԙx<|˷Mnu]y'MۄLx-h1ʰ<^_ma7ml6S%D lhS S+/ tj⽂V!Iĭv*Hb3^|•lf@ƛd]5hb%TUkvxؽֳm2;WI<0oP1%g&lVn}f Z"m"pD>9 fOF0Z=J[I!?>ufMaXّ/8ƲǸC2eׇ [ 1kRJd~y" YqnA+Ѭ ʠ"9G{JEBBNGY<>bU|=ړ~Uȫdk!= wx]&vsYK%v2)!˓ǻ=h#y'u$yKDY!yshzr* ܘIKvK$r%qBoބTKh@[9 W\s+łkyPfHmRFcģ莈OvO}p[%k 7Uՠ\^P,\9v *r7466WӞG61QrG7xϡU׏#A <#⶚JKUfC||=\<cGJdwEB]PO۫{m?o%ѕg*a} ^ʗ?`IƏǿ/xKbZ:Mߵd84x_ӛpfd~Td80Sv#yVىG|W&_w~3}w|e0:VXc<ؑ;=8~sɗ_#6ݗѝ-; 'W}1W/bz{RGi/:mґJs^lq<^9{.Cܛy޲ߗ^~z? z܏[Bh8-p?6gA|F{*Gn .x#s gD{"S<=4]+ʋTb!Η,lm rߤNZP{޽:9&o>) 2\po@0m_Vj 0L&ef P{5;{ 6Q6 q<#噬cTqi>H-hɮB<Տ<!snEdhCXA"]4p*IQu+&ZbH@U>ž, [M\PboNsW5>~<êd| /$ $f5mG'-DEѸ"Ԏ~46two!V# ~*R_n'(\ՋfY! 2+3,_:*x LɯWT@ ̀&4EpyaT`$<4X$%hq6󺚗L9]-bfܧ&ɶL%?7;tE"dIq=e5"e.;0rp&x3}.YwOĹ&C$[l ̇!3x=U%&lҍ{ֲvE-b6mE=yp[V;-D^eTf tMu*2u(A%!nṚͧ&7u OxL[3~Z5!e#0ƻp9ީ=_[VĞ8i{p aћr  jmOɔ윬6E䪻4fپ ?9bA{[ KNP,`o+ůMx0 nsv_ຄ(᲌xM .NG;Y[F:>`Rg\wa`v,ʈ=43a'5PHf)T\b$q7a?g̣b-($QEɺM^xZA%imwXw)^IqnvMgEvQ&yv3\@ad2P #M˺g4ïc7!9YosI2/q%yd\&L: <Ļ{>6gK1+d9&A/Qz^ ƗB⥬)H$|l* *NhqkA`h d 5s#3 ^AF?PZ05Q1DTq'So Ž,C"OPHEJw~E~)Ek3C 5n: gDi݄i 5u+tp;,&3jqy<mopr%.ɃyxV*nXthV! -gH3pS*yXߎO(--=DP~4U P ;,gz=uƺw:QQlN"(W[>&!+qd\!#Ƒ"(C`Snϸ3D[.~ DMwV_D;@'w]]]7a\Si"|Ԉjέa^Kpz;o!scl0ɬD U R`綑<ʆ" 3l8ODߒ1¯Q }(D\&|A"("7@jQWɫ:E@>3 "_ơ#k(8d{܋}4p. xWzzOϊ"("("("L@T$("("("(@L1D"("("("p-UE@PE@PE@PPCL0)"("("("ă*AKiE@PE@PE@PE &TLJ("("("( xRZE@PE@PE@PE@ U8)"("("(@<!VPE@PE@PE@PbB@1D"("("("p-UE@PE@PE@PPCL0)"("("("ă*AKiE@PE@PE@PE &TLJ("("("( xRZE@PE@PE@PE@ U8)"("("(@<!v"@IDATVPE@PE@PE@PbB@1D"("("("p-UE@PE@PE@PPCL0)"("("("ă*AKiE@PE@PE@PE &RcR"EC9,\^ ?t)"("(""F _c>tv d̓enzz|gX/*99ޣG)%Ґ_Bý8;z,Û@Q'd 7w&C2]IZϦwdKGX]=Htm9yYy[hf rlQЇp|B_('.4wD}"KbV8x|b0^xIj;s){=(Z2맭?Ӵ( cl?zdWEk6::Oq[h4..D:7 vhƗ5!u@bYvz"dbrB¡b9OǓbS1v4E !t_oKwAy?Ub|jeYQΠ.r M=d<[vVb,21rfپ<]xMo3v?L'Kʋɹ~@b<(X9$;,գ("0)1*<Мw|a! գc@qROD`d ry. F)HK3 }~L 4 ><߲Rkח@MY ah0'qU"0Ѕ^*8eJ E:fO`tmrOVJAb_'qWڰ3 l>,oG3V)g`/XNXN`4TuRQk?,lSM]jC!ՈxʔȀsg" mcnL #5E$t♓k{FDTؒ 2%z ̢˖б>z1*}hdT\T,Jkjb\2J5(:Ą _5`<#Գd"n砝+]"ڃ2$g26]˻0y/1PUSM!RQbf*S"Et1: Qaei*:%ݎTݍpXd](NPE|"!\Aw LC=fC*M{<΃EKQ{g,$DE&t _%%>p3E ]hھ 26\2r/fO2ʟnGDa|@ZRad29{R*[hM++ b0dh2rbRdY cطw:ra$OF.ʖ,BŽ Eki91/)Hp$Y|C^J־:my?w>>'kmaX;9FhA0w~ sinF(@O{+’|,",.A_Uwdb |v1\V=i~E$'9JObBl<[\VE>Rp`7w=.FlN([zxwNcga鳊hRnAY Zi81aN|4]>qsҥlczTbaE@P>p:?c_n[zqC&] &Y _i !++>@V2~\0r[ LXSK̿Q6Hsp,.V6Hd -Y iBQ1N0#ݚ̭jvH9J9`&A.EyRP -XAS2` "JyXspG t)0++–L;C/-A\yɅRґ܉F/;3$42FrJUb\%aeÉA&X]K,/gd,Rgy,E N)_,XM͘k*E)śN](+?k(_fȗ4ŎAb*P!Me bmb*'릏uSa~<ݍYyJTQV6)^.Jp\xy\ *̊#Wb7kNއQG(jAҦ`V,sڽHўR? ݈A\ S O^,ZR:] y{bk[HEl "N-YE ^n aRK a)}z,+׭V6̼|qAgKf/DyOkClOpbn|BfD9OWڬBK a2;cN畘 G pgeaxTM_oأI.YD\]9W]\ًOw~E@P>LkPhv`љ06]' iw˯< |sയǃ +{0;?B!1:;q]Cp뛋UzyQ.*XڶoG'i}eWl yCx袠 |jME@ :d@xy)yua̗o)'hJs@p^FWKD9e/ؾBpeE6th=0J6l-b{49|_)E(hEPnjí88#2L),Fޞ) >2&K|C6V0$86`sr90nl%ax}li3g=XnamՄL,BK`kiPp6&B I*Êtl[{R1gtZ4dR,)rc1O}9bsMKF:2pT"+far-礍:tA Tctl϶yP*ëh4eh(,a;i= nfADl(c}-m.kb9v8 m)3ڼk)>ZMmBKs]1!K91֡p^>Zz:bg=A٧sU9wI{Ob"vF'Y(nIB" . i p1{clBQnJMX A/]"[ bm("Xt' ثEK+`4<+('}XA{cn5bIB{Gal )`cKݞXW%r'Ruξq\{ Mh޷k ^n?a_睖R=h/\kDljx̝W+1Y呁typ;4`NzsBq\%Of"U|1 qF%NnENǍ q PNO qi}Ȃ<2xȟ̎r<0 $٘, ae W 7prjc& bRa5I il%0$4.g fS0+7@'8?› ApA.l9k*}m>1ֺ*U#l[CӁ};bX\!$wR`)DʕeT'b-Db;Rˊ( awk!VB\d1Vrڹ9IT8ov1̓3Q|&wPYŒ1Ii(+'W-E{gzk8\ٲ%gibَh CV{v"_~TxMZ_'m&RS3!ϑ/Ũp=r}%t[aŚ'Ԍfdױ~wu``n1\v_w):)|mH׿{[54O|Jo&eZ!'&tTSkQ}\(Z]f0#A;<#ӷͱWc(4њ}wؼ0B9\T sV[~m"Y,u5\,CgEo2'GaΒ/C<֊,͖a/.p~>k ̘EQ82N(r:^S`f[%HVEd2Q#=fWe{w1O4U-VpV^L&O]nG?EsZ>cqEqs|I }ԘEt wK[byӍTbPyjtyŵer37h QiC%v@Y5,Jfӹ-m30uۆXtE@#0CĜM%Jit6"#ʪ[G%48 $ؙr.!yBQ+H*w sr80V&NܟܻG5\ q]C]CS@-DSdc%K&}o3@sɏPv4ENcɀjXEs߈!g "ޝ)Rhjؠ0aU԰hF_*Tɲ3Va䆸G9Ŵnp -Co$RxjOL'ss1jLDk8ʃ#sy[An!.*BVO*q֍ UÝu3evGѨf8Nsp f񳴆㶨53+&[)(y2]?m/79 P(R,ogX1I3'2fgUh%U}B~$EgTggSL=MnˡzҢU#<37e_Xp%v2Gt#J3=OwM9>pйNŃ]<3Y<)Xn>ݦ ߣ`jX֘d58V{402;GA"IrƘMqh&-X#(nf<=e9y&?8b:;Zw3E/l[z/1O("F r+ ntgLK˗`5|g̚,.?fgD7gs9AqvElL#D6;S 4]x2<%ņ*935y%.h|ܤOS;Nyfv(ࡑ1cA3-dɫ(C-x,܃3 sYC6[\6D|*8/N/3C '{c:a)d].z-;NymCIq}: ]YfuզsyH,~inSMs-DH/v]M/֤0FY<.{OWϦ) 2qO/#jHpT@g[ܺ?5+^7U{ Kqc|T3my[*Y*錶)LdZm]įLlG5}uJcxX1scfbD_ΜE͐|g~>n7eqX*03*));lbNOkHnDI÷Vvu;5cGͶhOBZ2ҧ㭏"|Xn@3*s =]Nb/> ԉg7),}1לk?59'2?Edz ̎s!3] OPW(zaGށ.Zy4UyYfa9>+…ڑ4EMxy0KI݅(8>C"8o) -LMhl~aהB|ZZdX̸[TRU3/Fm>5bbZx<Ŵx|EKWTM/Z 4Y<qMa҄WBŽ+טD|/P[cN_[]dr3vXCTR ua I-nr6K*:d7,b$Ŝe_aV+²"{"1$Ҏg,<}|#ɿtWI}uoYp!I0EإP~cG6Z0YRt< cߖLx.m֩ m( Qы6);٧i7d S$o`/u֡[/K*YGоC`֝16}5v(eY?ٚl)%~e[lymk7`}֚t21*^p|p-ݪ[:Cq__NjU > csN7 d,'A$iRcM],i"l7x,fg[ 3;4*(mOS8ɚ1 qf;K[q6x5MAdnY7.ctH$W7ynvLO8a!xsn|`O 0S<5dך ߋQ957w>LѺW OƍPȈťC8@d?,ZS"\'~d0j^h2\F ;MLaoߨ{k|C|2 9&X\ϊeNġf2†4,TcF;[j(T,j X<ϴ"(FOѮpAӏhPvY#fy޸:#EWLN*zez浫d5T'%+:Ⱥg_- 7{DY/G) :$ *&f0^KFRۼO}_虷階+S~-+dg;0fAT7^&ͣx^q;5YB9 Ưz(crs1ʫWlzCZP\Vy.=KfHO^ev.㜯֯%Kw\2p'Z^D7<%*|9TM" H$D@" H$W6WZ^" H$D@" H$Ew*wJ'i!84g42D@" H$D@"  "p \^lV} ы(qpP;0-L8V#)JWsq LjD-wE]E%H$D@" H$%"`[Ǯ ~/;^~WWv~.|E|c!bE,+.o6DM3(rD"2ʊRƋ]h;H^M$`^/J*DC0hvc Mw!(!P:_%ͣ׀o2|&>PšBHD@" H$D@"AA "#O6C" J i$I\)RpQ]LPvO!pXA+Axj qBr˃6%鱙e8P2x2q)'{hӄ53a  a9˰D@" H$D@"pI%ppx"ODptBp]Oӓ(,d73}/o7/Ǭu3_lSDf)O|qB+TRY [CgD@" H$D@" !pp t&E ^ `hh`B6mqmUء^LNn`jZ%N&W( $#ׇr=8uʉ(BBS}ѱE8;|QW@rע]:ª戂EW?Ne\1B}zyB%6a:P~#6ԉ !G[ߌVyT6HƦ04@!B́PQ?Um݊P *$ѼZxTt:hhTD@" H$D@" \ GAxaq̸>*/dşϞYJ:4hAݍ$A#ؤ@<4a}FuU@aH1Lfqk&S :=vnv5N@BPɧhk"nA!o] ՁJ%[6/SPhޯ Uztmێz5:xYY9yꧽ U@yCmMAB}KU%D@" H$DDC])X S&3vnDR?NՇۻ7d1:6#hܸMBK>8fl&nh64#882[(Иa#MC| }avuEQGS9'#U jE#jX߭j E_O".4vm*x!"&3N-?t+qӓCx}hRjg"1L0C.j r6a{D5Zq Bv}B!c"F7c;KnD(A[ЮKhWhJ+d ! jҤibX@i`Hs$%D@" H$DrD xaqCuwEx˂DYN"&N 8MbCG#Ha<Cr#GXd~ P8bT:MA< C`&E(R3 b0M *" 23 #UW5,(pA?3L0M!uPRib"Y\&X -m]өg!@(\I.D(JCvA}V("ShniÖNfҩ JjR*(2/u"g Lc-^^X,6 S|nFުTKL(H$D@" H$;Eyg GFs"nlÈ*_*Η: 9Cmj8u= jq,T,L?)!PSe~ؕ$ZOYfPCOjйAK:ृbQتZzO" H$D@" H \ n*#т%rQIF K% ƿ5$/y兗jb^\ r+=.)Yz$֙{\(#'@ ztF7/ X 0Ȁ1 NQo749q%M"t Ѱ^؏S3H9ؾ[Pu0 ?%̬a0fxD߾Ag\p _ޱi"$D@" H$@A Bܐ a+[M @x`tQJds4֝S謦̻%Lq*U0:~3=/Z|KuC(F&w 6CLg|n#ìr0̗.`1DtJU ֢iD|Z55pʑD4]K!=( :`OT+[?ޏY_+ZuY#!Hb2JaW ,@SI7|5nQm]e:1p#ؘNmm@/\DF#%D@" H$DR!_@C.s S&nSPxt#eN"PwU6ҴqTq">b0f>¼ͯ3ڷ)UoGh!m3!LZUL`] a~<^auJ #`&pRqLBh.إ$CQt3x|[!rHff0NDxǮP_ XvދjZx3"ԌQڑm;ԅI/871xInLj,D[VY" H$D@" H.t}ockq¬Oޜk"Uv-|L+gHI2i )gh*̉zn.}B=!nj+2>w#VÙI E4W`hQLjJ>a.>b[4&XT@9-C=4̨WkzҰlj(М!꜊!i![TtF#Hf (I``q SI3ͼQ Yz4+H$D@" H$ cJJMh|T=dngoAMػ`Ir{QJތɈ+#Kg.# ,)DMr.*.CHa0Qe2`,8|ŕH$D@" H$D`ka%D@" H$D@"  ! ږeH$D@" H$EB@ .2D@" H$D@" |(YX5r+HduIL3pɲ64@5 CՍc:2IDSc`MEd"zqS5۠1;A4qqh6_}=*b傤7`~rwu{WӘL26WI:֚fTEBq}8 I_" H$ 7nk\z,իk[7p߯0uyе!|KDHOt+:UK"yEnCq1^.4o߅6aKHVډ",҉%;2^ s6D-%._3vmm 7dl #CG1-p%tbi)7E_g%!h2h8(Fƶv{]/MPǶ]*1S +A[vLje,7_.ncY} ~&KDB]X!P\n{ha0&5qq[mRc3= h; Qv[Ǣ# ,/"H$E@&hd!"ٙw%<[;ac8)NEhO鴈#8t#ƅ}p:=mD"6Xxj:xE1 v(.8dڪ*!h9QdžzCo8GJ/B˥LI!aC{/)r/Gw5 ͈d'J#XwxݘV6O9 <Șr̒>P Cao9);ͥ%gICZS88e{.۔Q>=6HQ7mlW:y {f0~dvP߀F8.ap.Ls7.>W" H$@ѕ{*{{18hBFsPU<ʔ`J|+%\6Axhz(}J᳍M{ BvTq6imMȞlUJ(eZwm36m%c8? _\3>ݥ 5Fdwבwg( RFcx&ؤF {>.pvy=WR!  4E1\lPeg}CP +VO.jd_DG$*hi OBC7(OXչܵhD@UaS#48B4P1 CRs:1呗OX@FR&]ܱMuMs;ٟ9! CF]M-h;BQ /*գX-˃6?pB-o9 :u+ƶ4&C&4&~;SN̨R£A:ThL>fvV|Էj/l9m5+=ØeVڙ6PYfʥ[cИBy#6Bx$wpߵaѱ#̝O8Mljb݆e#Ce3 @B޷Me}cۈ6!y#;B+siǦN+)T)֟C8E/TiiеsE6!l\IřKb2l;3wP MTz;EQ{cBn PisWD@" H`Rp&@{ eݷ :?^/+}炏^0JE@[eθzhS-q7:Ar[)Z89#~<u&l>U~\vi \wޠnn;NE,:yK@gk uovd D*u:b*4#iS6 mqjzdxlΧ*hkD] l:}iѹ]ؠG9mGx=yPnb[օ */ZV'&crǏV MЅ 1[]A t5jt~]n<̖>AfŶ\'M[M x݆&˷G64)7f:;UqZs+ =Y&OUiE÷~fo"li ][u&3;f"ĝYԕ ^A)Pg;[;lх\ӬulVֳ͆VJ8xEual2UY5U4Ig癹alRY UO Wmq?lL#t62q=(}&Kb3q 0Y~}/,f8aRS\Eps_E.jOMq!Nu853 'Uu푂 fuKzˆ "#ڣ\p$ڑ=5xmDpz\3dA|3D)l0`-iyJq}즶A d sۮJ<rS}ڭ:PS~ |<dy]x34jWUxYV7(^fVBz1VG%u6* 덫3T*@!DZlgf8fZ, ०Δ(̲sgܒeg5S_1L(]]*2jVW-ۆ]TӉ)E`cV[ 3AJ/D*SڐT[5c>JJwtjFkMiK$S6(o|x}1"F'hb5UPqFl}ݔq/9C 5ҨFLg"e.QZ)vBλsN{;`fXt!T6m ZA!+ޘkJrĵ u3jԛX2e}*"^,L_݇Z+X4M" H$ 5x]h!%nvݷPPӢyA^ūB!\q8Nr YpiZ[) 夭CeT{wc\ ,W LDG3K73\L&n&aĝB9<m]xOS5FH̉FjZ(d}eDզ$D:j]h!2 CYg']1+ #(hXn|e 61޾/qmjvc8lz=pcָcNܴ 4օpIi{Y;Ҧ5D{*HgLA6]ā_low8OuGctz@H9XK&ᵵ l3jp|wc|Qh8Ξc$k*!H$ Px]dwmPiӵ,KC7hإVU%ƧF1}{͒ǹkZygx@ŝ"c;YI } 2SFFiTPODdEyW{ >Zr"+$}5zf&v=!2V7g}QC8rG&Oa\Lq `ha jmŴrQS%vK3>~՚6$-2ZUO0 d0(lHHጶʶkp'65 ޾QZPGZ =AZxtפMHO ftC1,҄Vkض1HMϣWkvo3.0>y{F '-U7\LlfdgBkCΠQrȰ} `)Tqb'z`L[:~g# ,0P(4V(M^DKT}b~Isx:T:D@" \Tqop76Ww_pW>t|e+|֔ΏΥ5FfwVW0~f2tV`erpFo>S~90L>ۆkXSOcٯO%~sD=ޏ3ha6č{y6;_exO-nʏ278)p82x3S|o=bO3^J)1R^_1Rу:K'#+J~0݌~ lbqO׏ptlBfVĶV2v4D@" |p| eNo.%/`.=$]BUnCsG'>|gw\{&MvՅO]Y D+PYyR թ~Zz7t܃;u҉塐N|oF '?هU]$fd~i$Հõ E KP(ߕ?њELbsRsg) qnB%;? ¿yZ71//;/c=8:<}is7?-\Q) zv!H8a.|U2nXDc1/RV? O{jPۘo}%4w*CߴN^H$S.w*i$"p)n,–w}DK]lR *VrH0˟⮡!\nTs Oh% s>^AڛNAj&[ιM$7/&J̱qT\6Ѐk5WaBz8):0}1kSz=-k]oĈ!SFR2D@" \E\\U(&h:S?[?| ԳĞ8TgzfL| Ce4nڶN9!q!RH|h}>2X쉯>qqkō Wdޏ/Pe_ZTUDOIMd0,&D 䟊;{p[.9p0&mJܼM1SEQ+K~kyX,<vmFVST\i !9= 4GXŠA fq>OEF;sH{QN3)Wn2<xli| {om@ ==ͳ}0oك4>ҡ1~c8s8 Ǘ )eir`KR;X ãI>ڎq˿ 1F1"Km5?yN@ { fu|FñĩZ<8:QX/ۭXcRO`~~)ͶA/}IMߧ8pc+ocZ;kn-?mGr٧Y%?7<M1?i g0؂+38o3 k_D;v"s8wxԃaWi1VBmyQr~ϕ6fR}v54i=PUAV|Ïy]^UJ-܃lG;;=o=Ӈ)M=TIpvv OZ[N-֫Fq_<#Xk{(ն$OV8LXx'ھmAy(5|O"+9)BLoHz7w͜86v(ʿ >yci%&00G,O6Z3mO'>Q[} "⯎kuTᤍm32D@" \H2WڵZhv}zG!kuTגqᮁ~⌱K,Wtס \ G~Tg]24B[3(O[?_8 dtSkr]O=R~sYD9чp]58u/T}n 5&xmN{+p`^}Hsу_1EY-υW0Jz ]36=\%#v\ö1{v5 :X&svѶS꒾vڶ%:zX׊u:C8'@zWo Y;sʰb%m~KA&ZRN!?gU:7|ċ OS@0۩.߯+hQOL( {^[Xw'/w֩X]X=GZEcgޱ?CkZcS@ρgU;~W> f]zpýt7ǐ9ߟ쌵z$5?7D D3\zg;H>J$)pU?Bx {o +.<0펝q*Ow}ی/΃=5aC /rG{}󸥋 .x#>Cy%~m3OH0Q %Ι0i ~x%Ș]n}BC#=s 3g7vƩ7~Sq\y WdceQv (Up#;C͞ox}C5'9$4q Ob|WUlz PoךK QDB-֦haY[ 3{lG@÷q1~ >)Lc2Y&1ÚgU9v+?$>mP$ޛZ/>eBǔ4i%Wj+jcO?jY/,诮"l8# jۦ2Y PÿUn!_ tmx;iG lssａϿMg-g|wxuLh?Vf apw܅[e(:G5?G%KzLKm~U#G7!c?اKÌv,Ln U߅ƝYĞ.c<~Vh/13 USA7#=FLв?q*^{> ~e{.ӏ9w*[sݶ6sԚ}w[>8 -պHa;hP@ڜwlAH!6HX87K=!l8G]`f}8DZBd]Jfv5r;DZĂw'1OnlVkùaz6aayH$ 㸚Ju)BK#s} 7#@5\XYH˪VMu!_tw_|y.9V4|&F䢎:`vB H@b 129d~X&֢uQ]۱f3SK[[~#{[3NZ~zxB*!= f4$AA|k~ў;,c\yCxqgƲ8aԅ}.2?|ޫNz&检&4#t+lmX\0vO@ZB# gb1Hsf71Jqٮ#J9 O*Ǻ&?_E kܥ It[qj[[cr;Vj]>̄UX'VZrRa%Dj@ w~tIp6lX8s3,Sg} zgZEɳ_a8m]ᗽ #ZKf=^ IUsy0nϏWS~d]f W,?LA"`]s~G;0ĄvܤT)y 6MP`v(-4AS׮ mAiO>\Qx׎)]+mӞ0}wɹuQ^ ـUOџXJe+[C=|H0xؚQ]f 1;p }jE^4VY9IN|׈6jxOQC`Uy/{=!Cccf ޛH?jfq5KsXsL-p1j -QýWV%k-Is<~Lk_o"-'Om9ė*MQsLz;E q5[0f ,iҼiYu" =ł̱v!%IID@"p#gtg`98"X563^'4 2NO:*pRzNq=i\M_mO$Ə % F/T`wͰ7A{j̀/HiC8 G9W[9!\q#*3,}TynڢT= ywF>rǏ ռF ]XqZ?;b֌UQ _ 44'W9BOW$oBhP!x'F~ddߤQ3l;{1ڽd Khm(.m38N f4W0~,-C ~qλ l4 7EP^ =OCl%N5v@y%0,t.2 fJuK󣟦Fڣ}qŗ{x+Vc\b-U}tOH'>Lu[Qnj57{82]G'~4<0]pJ-88(m-TgCyVf-ƨ:.XxCW{xE u s=Ee9iY\-cܝ6"j-M"3'K<9y@u+~CX#Z 7 h/Əȿ^OA2/qWӾg Y!8&Heh⺚L'FaAeOr[K՚VT(r6 TQS[Ej\;_7ۼvg=go O!3+vnAs/é/Pi7Er1+6*P}&Z߂:=m!v-=<SƇBТ'AWϿhDz\Ǽ+ˏ6nA@-$hm׵Z1k4Uj PP f|8_n|-mDxY,ͼCJ[@iűØ|_+>O-8ND^ƉA}yZ+m۳Z]*as;?T~Gr؆xJ#o[5z'1֞C`5\KFTSUE'%$"/Çz[au+>W,4gSt=K"|X$ ʝwA-cF#=ΞƸaySҠp3ǖi! hQq]Ax(ޟk cA Sa*>cg]" H>`3r/oq-;im+/BE;33v& >4~U@IkN߉s絟48w;~ƿk9V"鐞mgpŷᔣ!x+93Oe<>mx1F=yvU3 Z9ӗ7 5~&ciԭWqC.hmdrX.v Z d)מ=n9]\ׅ֟ L۟=@m;;_)̄_~][4c_f Z̗_g2G/1-J[l33bmCĬFMXӼAYԿY3=/#l3i86S$%fV'[d#l  grmAb" MֶB쌓!=_ Z8ٟDJbhh.NDmLRS["#q\KjRʯË_;+y?'!¸E)J$5=2A +.[CsPCA ULVbm|-2qڦ c+$4wYT՘ nFt+x5%h]2I$0R|YtD@" *x3P`w=?xuR,[D@" H$KG@  H$D@" H$@҆C K~uk]~slH- A@Ӆ@UҔH$D@" H$R`@L֡Ip#F*mQn-,}a J2Dq2Eiq:a1yv7fx'kn؀UJ$D@" H$)pXzY2+q|kVY;;=E]Ft ;YVVKPLxd U`^w e8D@" H$D@"pE!pqWxy3;soWj'Rm OKjL](RO'Z?ƎG;zwKg#o[$!D@" H$D@"p)9K"mp#7VCŊ zۘcՊ wj?57㖟S#j>2՟cS)yD^0ͼCםy6VK+ȓ!%3tt|b !}IwC&_|9dkϞc ϟƌf j'̼&ny,yˉ%H$D@" \Hg*Q@YԮu 5+ᴆZ[W +[֡E:uh{_[ ycZ?{h#@0(#F)ͻE2&KfX3;NvO&T.LfM51jWj&:FFI!xPhyMw ~s9g7䤣!eHNIMeO̬$" x KAcOMSUq2ʘ0_^׽%k$@$@$@$@$@$@Q K6ޣ_/Ģͥڤat{iۺ7UC_QTW"N*5İWP6 +k=9oMhJP㌤T7\>@${oβՈ 9/t* * bAITmFrd+K"9NC>"'3 d8FFG$@$@$@$@$@@-ĤYiղxR8SY+ѽe6QQo)5 2 A C2Yݢ]Y!R3Oa{OU_#]v ^ʵ5- <}IP}/Fm .geGTO|eIJ,2-^.Ź"L Ln6Ilrgt.Fz@zo_H(x>zPy'S,V@IDATBC_8N0Y1MY:| 1;p lyθ@}C3"     IJ&MNlه+ĸ 6a|8jzPB -ˀ^t OƇ߱_)DGq/Jk?ieX|_ATF}szenHmDQpo-GNvi2 \}T8Ĩj@H9iVz\\j= vŀ]{Qy|*R6^\kR&bf"Zg]uB{ <"yB~JlJ+KTuCzR=}܈»O"ReG3xHHHHH @C,YoxS2#؋X(sKݥfƾNS| \x.]<кǧp88ىѵyf2xLwBAㆉzF$@$@$@$@$0|'>k(E/zF(M_ )O‹@M,#]G0eC;Q1}0V鎽C6j'zgۏöJmOy4^ \!5]p2"Qv%7{m4;=:}ފKpm#.7p&"%d,D$@$@$@$@$@ L[;mޙ=wC uS"U&qXjj٘?QAt]5HHHHpN*Ŝ lY!f QfPq}4R09j     %*bIq h8i$ @ PsHHHHHHH        iH!n,Ln܇K\ک\HHHHH`JaJU3;;`Aj^4~( H!xHHHHHH`<pHxPeWc.1uxrÄIHHHHH%e1ߺ%/{A(.%Hr'{eo <{.P.^Vc&q6c,g͂iu_[n;p{;<){{ze- AQ77L"zXJ\r*Da nn%̂AKz<-'o*qIſuX`B^r}]o-ES5c$@$@$@$@$@$3l DC qVSeK {db@ =0e}Cb[FDdeZ9Rfavi2xV,HrJY!iY,z Ӱ =EE&NKZﭫK{UF~}a2 33oYJ*!S#     IX1rqbaȠpٷz~KpCOL;D! O~1* YҚ ވ; Em.|!q%£ũ1S3kV6v?OO\pd[wx؀ˮl(X $ܲ/?܅w?9ebAqG(Dl3F\~9L$^|D'     %*F bOm!z_hw<(XVVt+A^3*ȓǸX/'ĵElp0 uq'.C(tW/>σ~FO"\e˰@3zњlӿR %_,/`|w?,/ RΘ#b$06n'*0"󊕃 2`u.ֱٶowt<艖-Aa; wQl%T6yK!wS;r f̓vQHHHHHH rh 18 ')| jߜ;DKH^ٛ_Pa?oF-?息^~xn?tz1Svq(?<"^!     D(D!}@, xjX>1"| 4pdcwBO 9%@ U/)X ^&,PU.SCWLYr#'RdΈ$'     XWTƁ@܋%CdS4ַy55/)΋ʗ 5Ƈ⟙wsOÃ}{?-oXrfC4֪L+勿< p<#     &7 0~cB¶]K~_> DG#A~k emI-0h3&z 3/K5,|q:6Ko}żW!2xF$@$@$@$@$@1% /18pd KpExdB0%D;R1K,\2ZDj¼ȼD-"ńTlLX cFHHHHHHH pHE  cFHHHHHHH PH$@$@$@$@$@$@$@c&@Ø2       @ <%wcA,|ྌKgP[&`iP$@$@$@$@$@$@$0 PZeːgnW% IHHHHHH!0MWI=|e7R(8҆pZWHHHHHH&1Z8ĨrWl,LCDPQHHHHHHN:#s7.TĘ.#     *puMy}}yD$@$@$@$@$@$@*b]ɖUYXHHHHHHH`Rऑ1l,#q} ,ij1FHHHHHH&1Z8ĸrgja⠢NEK3ft$@$@$@$@$@$@mc\G) V"ψD%vå*bё LrT8ĺTn| \!ք  !1$/LsbLё LT8ĸoPHHHHHH:#@8F>+)lA 1ѐ #@CL*l}o5YK L=T8Ĥ8I-kzk6$HHHHHHHH` /S(*  I#@%1$@$@$@$@$@$@$0P0j%      )@s8LJ4Y)DB(-UJtx' kN8q:ʎ Hs$~5f\x~t9qB {(5_DOYr5S|iأŋpcq([t iqKfpGڬt+Usߎ/|~6?uHHNwj mC~vQwDy:ߞ]9) @$ph+H;>tDqCStI`0U{R"g>V=GuBt˿4wJIX7]8q,-eHs/'rU;Vu(3 k]T7u ilQZ4X{TgNqszw9RщϾ$:犇誩?WyJżO>`3mwϫxnJJ.8yk\kP4?ޗF9|t}>p ݮ1 Q8ۼ-\Te(cc9ޣSk*EpB(L5a؈BL{}.wq|WcrB-W۟ǥiێ+^KɪB;g'(IybM)%w;1MAL 2=! 7߅+bu.YzfUn)Yd 眏+ *]8j>x]-16gT: G jV֖S@)xlPG` ʻy Rs IqD>c&(*Q׮,|kF#tف+=%N`{gDgx)nZ"J1;'6ʆqf>voX6})%[.VNƣ;¶Ua p}!0o #TˊNk)1Mflv>GcS/;1, X–-b|]9bg?JhDB+BzB6ۘ}kΛf ^XDp}q;B;9p{zӗcv~8pOƜ}qBTϮҞ+Z%th+mt߱y-3=CmhW TyQFضjK7k#1#  tYD}O)tbH(|sf땺:xd=/K7rA)K}- -]CO>ZOysQr/KO܋?ͻ^5Pemǟ RGe@ك{m2lH3W 7W-Hw"`Кp~K!_6ҩ#824AVâJ0M(xUH]w;^G:+:/L-)J!؊$1 ̗6ə_df+㑐nHBDd!OW5ğEͦwFsCD{)ܒ+:.T?n\b[[-l^z~YJ2sŗDm.{} _ ?A*Hvyߍd9H]r#   @T8o/i)}Hu>/w³Δ4MyZR,G2əG#],~h>YIhT(>[9"3,vj 0XVp|{[^ĥ#Ux%XHchsXW>`p+ !WJRj|O/+ra񛪬.c o9o'4>ㅯ~͙oQcuQr!NL $k3 25A %j IޮpH*2jW ɹbNnqN'!UX׼f%ɼr?=sEb fTGs$K Nh0F3Q3 FY^[+ڙRn9dH-g&Q~~̟Q|.mn۶0Mm-`R1MK4$Ȱ"" %p`lDdl$Q6nZJiErivB|zSe̅󪔻UL__d^Ne֙$sH{T<2Y!ԱE 3KRb2+CQFҐ~Ecwe^;/o@rLKM܉9­06D\ ^77y=AZmqIP 5 E᠔t$@$@$@!RRFa6~R&'}<HXMh"#{Q(ǴWbODz#Nw{ogjjy=B3+3gOeFfN/D|܉AA^Cܚ{mʆNIk d"4ߥ?bՏ=6C؞Makp>ЗPŸ+'n9\MKߖ㨪^9PjO04d?Z&mG^Ob֔IVIfoJ#3;qNڊMCzg'.J;'(b(4gmo}!ݣ F³c8_KcNQ8CbrK> ͝t{Ʀ 0Ҋ{ѧں>kyĩrJO~U;v!!?̴X]Q;{4/ {Dkם[qr#%# m^m](^c%]i0FY52^YOCD[r7h7SB8|1"j5sy6&)dף_͚Qk'ƽ*>)i:y芾KUCp_!ѨS+o  _j^ktHڐxR`dH(tplrї<XʆvԼ*.VaE@IKU]FYlsp&|sGBQ84↍>eÅuK!JD᧔}*Qm{6t'C!V7t$@$@$@_r};V&Ĭ|f,&˄{3y㝔#7~B9gbBZcr*Զ3O _p N6O?W>Tdɹ&իً$8Ĝ^zs %P i-.~KbѠπ~񄘃[wIFOjDh՗\C0^) 1PP>^gK_[i)o|!* JYՆզc"[#OHu2BXS2h]|JCuG{lNTƊGLȰjz MUIz) K _\PC 2$KP8LbdێQ۰ QF/^)`(ts] %S*O[ũp%teR2ܓn@Bzm?)͎{(?XlO}(kdmۡY88O:   H P)p)E\==9G>vG~Zrj}{a]hH>?D]':ʆ*|.jKs/Re)|cHNŭܠSDn=twQbXsrPՐ݉0 f̣w7syp~W Qr&r$FO ;T}35X\;&QR{&91o6}90%"l/6ʶmn}3ke/eqd>[SQ21 "2alm>N;(Im2 V>e[`l":EE̬QBPsuVeS#U0rwLi_ LL$WxHHH`8~ zb`**P–C"H'$^AD$ڄ Q#G٫ԯKVUtnG ղR*OCb4^$Ѝ@Lku'sZN ʄbg!WLwSe"kp ̘QWGzԇVopR(GƔ ?P ȟ'#ʪX)ֱOϗqCڌFϷvEMvD $ȖeGMרrVey2h<bZ4R=ʊyݷREcor\^LU x ttfϭ/=92XH &|GѶm 頶䢿]N$g\'.-G $@RJrြ[8rk>.f]j ܅CG>SCv!Xb¶ ïz_%p oftHCT2CGX($< Z=H=6p#B|Jkw|++U#.]~T=хXU++|s1am{z<}mr_#jG.k,[.yH$@$@$>/G@+SwUՄħ"esn.vfqٷ]VІ KCz<2#(x-W-2n] C82WY8HW(޳);@g_3zo4ė&4=C>efZ#'w[ϗGso{;Rqd>  mN.qF0?ed90FY_7M{NMܖR? Rх6.OkO"CzhsL.g&ڀD}2Q7 >!ȥtꂬGm 穧QQ(H,Q3Ir])*93=/CH]?);mi(3s ea!Lmbω`y2p>wPI_{^\9է SE6Ĭ̦PM(U2狴!0;HQmRC !+6& [p҆dۼWds>QOjP/DdI^ o]R8X\P𩃪NVf/IfƖbnMEVh#kۈ54. ?*y^,/zɒsKݥԅ Ӂ|:QZx;^NA%2Jo0nzwF8&rSLfnwY+F=J*ZmXr *}n}\MZ-[eΘ3ɫ2<-})X;6~UڱRs2-@d7 MWY`;x˦p&}]gk-Ki~PPeR8}#%S5D.]8Qr0@6D&?]FX1^+lC!Kzz !1c ˊ +W( UGLk([> j%Y3UM=SM"){T ъ)anK@34%_EG5{4mن[}{аm}˭y+ %o '0702>awԅK5AMF0# 1~ДmE.zT*2&}J|vӖDP;ڰnՃ=s^k,ɫ7dė,7K{*qLGF?nȪ_tzdXyIa#\='maF%nhBݹբtUρsa猼,BuoKA8vO7WYh69X|m;Ի?a*((0S N굞Qi̶Q;kXu>MS6zsߚ{uf ھK/Y4f{{_oҁdkȔXSY{6 2._׶(.\>{stIϾZiEYOm>h#skJҍmHN\h8~a?}#_]\ϋ/'5E22|GV91Uʆ>{6޵ ~d'ϱ/mPfXE8-: xy4>8]LHc0M4S"3 ҋ%&7(Z琕!⤷k 0t5Ej#v('Bg%iK\1]vL|~#FD1iD;oxZ[/6g zƸtNdB˚ _ nG4{lN8id5|&ޗ?B i:z^_prx{Ua3 uB'd5!QDy÷Ց oAMX!NJNcm9 8n _baVߧռ/W%7@9ԤH|M.~?!m+K/Ɂ̷^ԗUm{;J*˜\dd%: ގ5ѐFxF93=oE3v IbȄxHH{ 0Lu]8?e})ĈX4_lhDx֜۳$_v/y^@*c"!E.jE3a9'5g f#' ژmeͭK v4H{k)-Esem4 3r;HGh@] 44c:CoIY5[HS6ؔ z)ittc #0jܼ\;vy/^:o?ۆLz|*6e)r(mh1F΍<ӉL!0M>ߊHFڶ?+iGeb@MUyHHH -x8U;Z[ĉFPDM89%ϰ@$rm|*skLs NPV$ªeb dGVRȶe%:$ge% ٦ ĉo(y9˱JTkD$&{z6c#F ⶭ_Gվ% $@$@$p}wbK@Ĥ Fb[mB bF#dIG5&F$@$@$@*d1HHHHHHH`2idKi` ;\pF\jeHHHHH&*b]YE(޼Ahx^/r00 кT Pbz=L=%_RYn:^ߋ')?&ٿ:R'     )H XWLtTey/òL?g!8IżOKtŴa)yZ͔UuSsuG?-+GT8X8x@$@$@$@$@$p !E/zCY+8US6t_zѯ4LNX'O.ZO#+&j|qGK)LeZ*lHHHHHEM ;u#J%`#eaSϳg*ew>)f13"     |pQĭYE[U#3 qmc 3PnZ",b!fo{ Oݩũ<[Q_}yťfb-)K}- -]R\'N>Z{^v,.ҙg>5Iz2 H03&c'     pQĥ4ؒSfcnz}%it}ti/#E۷܅3Cv5Y 8fbb(HHHHHH`2Ib9S)o 0҃@FS/;uoG|Wų jabb/( er$qQΧ̈&r7ک)eEa|$@$@$@$@$@$0L5 >1ٱWq6a|8jzPRHCbЋi;+6(^܁EIb'ñ +:j6o\OCQ ɦJGV6:     V Pu "Y=qsӪ{c,8{}tUGH\zqIe.SjfuU 27DL* 5z)uf,ESq )뉦xg;hCmJO"w$@$@$@$@$@$p !,7w<`ͩ`F}Ϗ}~SrW {R, ɹR3Yc_)>Niw$@$@$@$@$@$p-ZMe=MhpO$@$@$@$@$@*E%ʲlM^vt$@$@$@$@$@$p5W, \}pu 5G kJY       pu 5Gq͕ |[JA]٫ HHHHHH&-&9SGwHm8èIHHHHH!<ޝbܶڏmyH$@$@$@$@$@$0,@f ?_6l:!A6>qxBjU3W *^QrM xA6RFlM]Ȧ** ]vT<)nMYP|O6Zd3+6I8?өUq/0)Mf"ܓ 8pC"&%>M%R]q]T%x+S)Ji`wf)+ʟg:uMݙy W{4Me ٕHR<Ќ̗J'ʓLq$= xH' m-۷e3julz\P{U63vA2+ɦN?WL{8u_z3r8SJF9ei^6~d23yQoHHHHHƃ-ƃ*J)NT^jS)+]) blq-5e}R Ӛv{Ce*R@]vu,M 8a2P :{/ĕr!3D{ϞhF{J`W@4CY`q9 *Ƌ,9%\a))!%CN j@ P3Iv*ʗf/g$ LT8L`>jM) @6%.{jx(ggAK,j)%ƉdFyaHHHHH***~&n'`WߕP+.B6 ?Aͥ8'eS P}fͽ~0ܓ X |C"X#axj4R JBѺorb9eᠬ9T9EFRE.ё '@ä"fHHHHHHzu '@ä"fHHHHHH*^1$@$@$@$@$@$@$0 P0髈$      G1<9s|d[e 3#      WQWH<Բkl<$      TĨCX6e\lwhGG$@$@$@$@$@$@*bTJ<˦SC,HHHHHHHz$@Ck-1IHHHHHHz"@8ֶǁHHHHHHGT8cc܌HHHHHH&3*b\;Mi٠pHHHHHHH`!5.T8ؗ˴! \lɮRRA)I6U %      E q$m_bb$@$@$@$@$@$@$0pHEC 0]y= \gpq/G IHHHHH+RV)L& IHHHHH7T8Ĩx"(0'      1p5Clb !      )A˿L2$@$@$@$@$@$@$@S'2UŌ !!SS 뜀cf%p`I |8פ^tW @L$@$@$0pwP9\eӽ5 (_܅cҜ:I$_E;=3?]sy8t텃}| G_#TTT)xk>4Sgc^{22a(=*_wѼ gbFA$@$@p̵3f}C9<Ɏ+܉[5ф4T|Vѻ˵|RY;ѕ Jq`ݠugI|A%Nàq˯nmqČYjmmɨ8j` d*GԵ瀷T$#$ъ>uyq܋*ӏu{砻sdxKg2qㅾydkm<!)Al{a}n3mo%yV% 4eqѺ~D-I/Sل[ *K?9[*4ժMXV stWgJ0_S1,/NOQjl9i J엷bQ%J4>!a_N(BdӲjʥ~$bfR9ytt*y8j-uӓoӚ#F{H^R 3KJ1x/e 2UXR`YR ֺJo;~CH~\c zN~Z Dy'?"d=o\el16T}/n|g]/nwqtW Gh&t$@$@$p@ʸ Ĩxi/7#i)}Hu>/w³Δ4)U>.5wfzs,~ůS>|. 3ep£9>[-%"ZBz$sJڼ%my{"fo%lǫGZyDBӅ[ uT8 IϡR͖Rvz]ِTewexG[ SⅯ~͙[ٽoFxri# 36 ,q3Ot$Ix!HOX{%0[sJh6= Rܸ}42@|\2ct)XQ IW&WG<_ǒ|S/+,iG2. !mqzV?>jOڙ!D cޛiB=o+;Rs =ZsaYۛ^/\mxZQ9sh~6}M ՛-б8_KcN?喐|fAx;Ns 9ݨ+Ƣy$TVbPa ~."H5l%H%!U:KL\WF4E ϰL[Wq^'!NƱy\NB >|r*'. bMmߍūT{G2|(E{fnĕcajO阽6JQE̖2 Ag:YԶ#}lێBC&| "ZfK wF2!P4e™{MS $avq"Rl9PP.'喲R^GoJ6/$~ k`'ZҵE T k3qyBdp>B$@$p-/^ jQd^.e$~U(f;)G FM>r*CO<-~˽.8x".RM"<+vmbN/=bz#u)k}cz%h'(Fd||OҎ/=˅APzgxqZ)Q8cOk=|2dCVy*YX+ڐNtjǥDYu!]?QdX T,*Pm[,qm}&dL/X8.yF~Z.TʵD\" )CN*igC22vsT%M`HB*iha&a;,|}ۥ)KehO*5cۨ?:V[yq&kzUv^u/jȚ'u"V2 A𢰏%6NNw-`Cfn_sPb v$pm79'-kC툾SbUSh'Õ3)dťGK5kgIޱJvLΓaIz * Om@[ojk(!pHPŔrjw. @7FS GۡTuq Y.Uyn0tz_8YeȟN]@ [0ZC~7Dgid'{%':̕~l~a|QkJ)q]3 m @C,-RdHc*͑\'(YA 1Ä4$K;Eq(t'c_ e2嘻Z܌I}~xRR:zh)HҰ~WG8y@ƚP/uۈ-1sUKϹl%@>a2SC>Ó <| )7$KvAywQ8(lYvtjJ(lӨ'F:~a'bmܱ={d빓4;PcAxt-V|id27Cp nu5ƥm+˛g8mҐ %nSJ]LeƍTN=h"dzHｴ1dZ֝2)Xk9՝_:]Y 2cwV1w;8>8eCZQn/UFBr;Q7 ʋ1)I7߮fn/|N㮬Q{Ame$r|cIHH`<PvCCzJD ވOE2\$ c QY$;6gNy uYi.xew! _>"C82WY8acRcŔU);d&5(s8֣~[_OA|__hEm~%sc ҕ 2e=IneL)ČFC^T,j! m%g~NSriaH%41]ԏc6 tyjFm _Zz\/wX2}i_jطcS]Z:o ^[ҩ/9)QxEyjvT$#N]!AO22Hwb&~T8D¦n ݋b|bbMջGn9>rĸ[ h.3^ro:j>w*p=d])Ȅ|mXԠk7ݱ՘m.m{ }"S^d ˯Cp>{G,E_?V0GYiF kSHzJVC-eA?]T?)v{t ͎{<:JD8幕HB#i +TQ%>ufF5Zpk\#'rH)Xq9݌ wsZR,߱T'cz^[e>:׋.W 5Xsa9 h-dMpeGwO&\hYzl^KpHV{y7:Da2lhly4\,3(yYՓ&_:*Ïd5tmGd'ƅҬn*}7}yKߍ'v54Cއ pQC&T] G" [=zυ4^>j{0]2J=Z`;NϦ5ۜ< )Ձ 5X%QbP"3`ޢ6vg8ʶ hߨ#IU];g5g؞D(PKޟv7Aho^WKkq}`է='%Q|mm}W.m_ > Uo SզgSa2>;w!ϰjܵߺx0|G $@$@$0 |C2ɜ)7OiO2DaY/!7I |)I)jiǞ3)PS({ocpvd.i4Ф#ieVYJV(V)ݍ],T JP-J Q?lI1I/WvmGOgx<3q=svy苍zڝZVtrv(oppL_h"_[Rʪ$_z`T!c! nz^voQK5?K>x7ٲNW9>nL/^%_y6_}Ǖ׎\M>ϻ2uC=+VS;fܑ!;}$G7vG%)q< y{ߝE?|56 i힤ci.ntZ t5|&jꀌ p?Ɵ{A9%P4Ϧ=`CPdY-Mrcj,wVcbR:6| ~~9CsyV$%{X&kfȍbG[N/LB@`as}qyǮZ z祡]uy4)@@(<!=kux+HIDZJ^/n\i7K*7e\-"D?%E2-Hw4yB>|1?Y>9p\le[_yLn]cѿ}O6lZ&'k֋}AiT?{i%&'h0xg2_`ĵ(;N   @a pUVi1-.M]Q)U2UR.WKGlG]6o7Fr{AyX E5ښa[U)8D.E󭈝(%+Jpr\Dƿm ؤf>UzXcI\6yuXW.   Hy#\!%2|Ng_"o 9Υ].kvn U@AsD48k 1 AaȸGYioIϴ!փ/I^kar5LbaSe9.RQ{e"yv_ȊKhR/Lɺi]/+w*+%GiWh; msr2(@@(d< 6 n-HMXQ,[wD3-cc8 7IoHE6fٺZX}[%O,+K@@Mn鮦^wkd o^bXoWt4dC 6 tO}@OF:]CuoOJ2iO)Z/> єpm9B8 @@(`snpZ.ڍbN9)7d?3_i],~q]^B@@f4؞.ٸ7@@@Y$@Y0Uh-uEB@@f-f#@@@8p8π    08̺G !   p8g@ @@@uYwGМR@`mF   a͹ |A^,:   ]Cv0nM@@@i`Zi@ r\kP7t'%rK׺<_ֺu[[,=u.v Fܨu)ŮSF.X9kudZP|SKm!]&ӝz%ݣ_ 9.@@I,Bs  :[ש{.], ʵ}%[>;϶[jَt;.~O4C구N'\:.V'\2] 88)   0ESQ(e"0Ywguu.W^]Xr[zLw^l8߫gt΋L/5].Ń:}R꭛& *9ص\mY,.9]]>   Ta*T)sRoYP ^mmi.v-b^݂aZG˽}eך;<2~?x̼ÉtOd@@SJqR-[p![rAly=!s >fҙ/;@@@ 0U\[7R]%ڽH[K|[aMEr˥l?`h}}Ry@@@B P(OzHլ e.rnǬ{=c Xpɂ %Ep*+@@@ p8\n^ݰ͸`]" ;~Yb DX9 姫[[D٥^ 2YaVdIFه  L@TL+{~m"EZ2V@q>K~?]~^{cX+:]tq;5   phi!tX7 4^%]]& ND%k`9>EFjPEw@@@ P "   0R15F@@@8#   <3Qc@@@ ^C?"*   ̼*foj]MSh <uEB@@@` 0KEH|+˦m&   sI.!=_k9ob-,?@@@{Bz{{tKł   \ S+f@@@$@a @B@@@`. p§^9eS4   ,@!Ӭ幖 6C.| C@@@8:6L.;&   V 2k,ݘM xxTg.\@@@%@a вY+R   @A Х"a]*\*q@@@cB~8xl"  )T-`L)k@@@&@!'~tE6X#   08 Z^]B    0#Z~2#jJ%@@@@`0hyTT@@@#@fqM#2XFڎY|   078̍\wPV96OԷ2 \'IoK"    (@JJ)sThiy`kFFR1-rT+4>E+1(|@@@ L1 $S&+n7\*vP_i`COp@?hgDў1Y   P=sRk^B }.=uɉ[i*\49؅   PB{* )+#2(Tɧ7T<-q}I_&K|~Ú1|~.KӿItv1Ke/Ȱs謗D^o^ot5<+G;'|N==vn?u]Ӌ&wl쓓 eɦYcp,{R5ϩ!Yԯs^Gɿ^OK,P w[v. >,>w/P   0s8eqe|rW e сdվ]H,^Q-%WxR*WLKUb`lEwcPb;V|E(׽Ruy7:oU4J֯J/"XF6>xD޼qy>i]׫WR%=M^ݶ7~i.qiQ^   3Q V:k_`uN2oޞ2T\VꔏJ{k"m׷˦6| |GW7w,vf'G-]}n6 KUP,B/="VZ<=Z'Ɇ_/ʪ7jۊuCtCe[eZJB䴍rXkwٹ].#E   08L:!on._mUTW%VRL۵]tv7_yk[#dڡ{ISuy)Rڅ⁦zh\>[&ZcLפe ҽKZ&+7X+ .}1~=z*Շ   3Q/:ww`d(ي`Y0f"Crcȵ pԭc CԿ=>w=E%_t(2/VKa@@@p8Lų HXw: vwT t҂z.E21 [AՁ-I   -@aϰNoI>~P|̅Ur^ CD]TtR!]-zljV\"HM 86K\Dʴ.%"%9:tiʰ8 ͳtK4zKF'V?@@@ A4>W yAr7zѥ|<$kZXa\vϽciK>$6rC[Fh)L>q vxZBm,gwt(=Jtݶ>6fٺZxvIS6% @@@] V3.7{-]>X;{c$lǰ?hGDL4[@wq>@&׸@&ϝ9ux;} N@@@@&&P,ޚ,2rS    8G$%6et~!'O%FsHg@@@@ U m!5@@@@|OAE@@@'@@@B %!    $X#   @hB @@@pk@@@M&lrIENDB`Sia-1.0.4/doc/whitepaper.tex000066400000000000000000000765541302033070500157010ustar00rootroot00000000000000\documentclass[twocolumn]{article} \usepackage{amsmath} \begin{document} \frenchspacing \title{Sia: Simple Decentralized Storage} \author{ {\rm David Vorick}\\ Nebulous Inc.\\ david@nebulouslabs.com \and {\rm Luke Champine}\\ Nebulous Inc.\\ luke@nebulouslabs.com } \maketitle \subsection*{Abstract} The authors introduce Sia, a platform for decentralized storage. Sia enables the formation of storage contracts between peers. Contracts are agreements between a storage provider and their client, defining what data will be stored and at what price. They require the storage provider to prove, at regular intervals, that they are still storing their client's data. Contracts are stored in a blockchain, making them publicly auditable. In this respect, Sia can be viewed as a Bitcoin derivative that includes support for such contracts. Sia will initially be implemented as an altcoin, and later financially connected to Bitcoin via a two-way peg. \section{Introduction} Sia is a decentralized cloud storage platform that intends to compete with existing storage solutions, at both the P2P and enterprise level. Instead of renting storage from a centralized provider, peers on Sia rent storage from each other. Sia itself stores only the storage contracts formed between parties, defining the terms of their arrangement. A blockchain, similar to Bitcoin \cite{btc, btcdg}, is used for this purpose. By forming a contract, a storage provider (also known as a \textit{host}) agrees to store a client's data, and to periodically submit proof of their continued storage until the contract expires. The host is compensated for every proof they submit, and penalized for missing a proof. Since these proofs are publicly verifiable (and are publicly available in the blockchain), network consensus can be used to automatically enforce storage contracts. Importantly, this means that clients do not need to personally verify storage proofs; they can simply upload their file and let the network do the rest. We acknowledge that storing data on a single untrusted host guarantees little in the way of availability, bandwidth, or general quality of service. Instead, we recommend storing data redundantly across multiple hosts. In particular, the use of erasure codes can enable high availability without excessive redundancy. Sia will initially be implemented as a blockchain-based altcoin. Future support for a two-way peg with Bitcoin is planned, as discussed in ``Enabling Blockchain Innovations with Pegged Sidechains'' \cite{side}. The Sia protocol largely resembles Bitcoin except for the changes noted below. \section{General Structure} Sia's primary departure from Bitcoin lies in its transactions. Bitcoin uses a scripting system to enable a range of transaction types, such as pay-to-public-key-hash and pay-to-script-hash. Sia opts instead to use an $M$--of--$N$ multi-signature scheme for all transactions, eschewing the scripting system entirely. This reduces complexity and attack surface. Sia also extends transactions to enable the creation and enforcement of storage contracts. Three extensions are used to accomplish this: contracts, proofs, and contract updates. Contracts declare the intention of a host to store a file with a certain size and hash. They define the regularity with which a host must submit storage proofs. Once established, contracts can be modified later via contract updates. The specifics of these transaction types are defined in sections \ref{sec:contracts} and \ref{sec:storage}. \section{Transactions} A transaction contains the following fields: \\ \noindent \begin{tabular}{ l | l } \textbf{Field} & \textbf{Description} \\ \hline Version & Protocol version number \\ Arbitrary Data & Used for metadata or otherwise \\ Miner Fee & Reward given to miner \\ Inputs & Incoming funds \\ Outputs & Outgoing funds (optional) \\ File Contract & See: File Contracts (optional) \\ Storage Proof & See: Proof of Storage (optional) \\ Signatures & Signatures from each input \\ \end{tabular} \subsection{Inputs and Outputs} An output comprises a volume of coins. Each output has an associated identifier, which is derived from the transaction that the output appeared in. The ID of output $i$ in transaction $t$ is defined as: \[ H(t || \text{``output''} || i) \] where $H$ is a cryptographic hashing function, and ``output'' is a string literal. The block reward and miner fees have special output IDs, given by: \[ H(H(\text{Block Header}) || \text{``blockreward''}) \] Every input must come from a prior output, so an input is simply an output ID. Inputs and outputs are also paired with a set of \textit{spend conditions}. Inputs contain the spend conditions themselves, while outputs contain their Merkle root hash \cite{merkle}. \subsection{Spend Conditions} Spend conditions are properties that must be met before coins are ``unlocked'' and can be spent. The spend conditions include a time lock and a set of public keys, and the number of signatures required. An output cannot be spent until the time lock has expired and enough of the specified keys have added their signature. The spend conditions are hashed into a Merkle tree, using the time lock, the number of signatures required, and the public keys as leaves. The root hash of this tree is used as the address to which the coins are sent. In order to spend the coins, the spend conditions corresponding to the address hash must be provided. The use of a Merkle tree allows parties to selectively reveal information in the spend conditions. For example, the time lock can be revealed without revealing the number of public keys or the number of signatures required. It should be noted that the time lock and number of signatures have low entropy, making their hashes vulnerable to brute-forcing. This could be resolved by adding a random nonce to these fields, increasing their entropy at the cost of space efficiency. \subsection{Signatures} Each input in a transaction must be signed. The cryptographic signature itself is paired with an input ID, a time lock, and a set of flags indicating which parts of the transaction have been signed. The input ID indicates which input the signature is being applied to. The time lock specifies when the signature becomes valid. Any subset of fields in the transaction can be signed, with the exception of the signature itself (as this would be impossible). There is also a flag to indicate that the whole transaction should be signed, except for the signatures. This allows for more nuanced transaction schemes. The actual data being signed, then, is a concatenation of the time lock, input ID, flags, and every flagged field. Every such signature in the transaction must be valid for the transaction to be accepted. \section{File Contracts} \label{sec:contracts} A file contract is an agreement between a storage provider and their client. At the core of a file contract is the file's Merkle root hash. To construct this hash, the file is split into segments of constant size and hashed into a Merkle tree. The root hash, along with the total size of the file, can be used to verify storage proofs. File contracts also specify a duration, challenge frequency, and payout parameters, including the reward for a valid proof, the reward for an invalid or missing proof, and the maximum number of proofs that can be missed. The challenge frequency specifies how often a storage proof must be submitted, and creates discrete \textit{challenge windows} during which a host must submit storage proofs (one proof per window). Submitting a valid proof during the challenge window triggers an automatic payment to the ``valid proof'' address (presumably the host). If, at the end of the challenge window, no valid proof has been submitted, coins are instead sent to the ``missed proof'' address (likely an unspendable address in order to disincentivize DoS attacks; see section \ref{subsec:host-protect}). Contracts define a maximum number of proofs that can be missed; if this number is exceeded, the contract becomes invalid. If the contract is still valid at the end of the contract duration, it \textit{successfully terminates} and any remaining coins are sent to the valid proof address. Conversely, if the contract funds are exhausted before the duration elapses, or if the maximum number of missed proofs is exceeded, the contract \textit{unsuccessfully terminates} and any remaining coins are sent to the missed proof address. Completing or missing a proof results in a new transaction output belonging to the recipient specified in the contract. The output ID of a proof depends on the contract ID, defined as: \[ H(\text{transaction} || \text{``contract''} || i) \] where $i$ is the index of the contract within the transaction. The output ID of the proof can then be determined from: \[ H(\text{contract ID} || \text{outcome} || W_i) \] Where $W_i$ is the window index, i.e. the number of windows that have elapsed since the contract was formed. The outcome is a string literal: either ``validproof'' and ``missedproof'', corresponding to the validity of the proof. The output ID of a contract termination is defined as: \[ H(\text{contract ID} || \text{outcome}) \] Where outcome has the potential values ``successfultermination'' and ``unsucessfultermination'', corresponding to the termination status of the contract. File contracts are also created with a list of ``edit conditions,'' analogous to the spend conditions of a transaction. If the edit conditions are fulfilled, the contract may be modified. Any of the values can be modified, including the contract funds, file hash, and output addresses. As these modifications can affect the validity of subsequent storage proofs, contract edits must specify a future challenge window at which they will become effective. Theoretically, peers could create ``micro-edit channels'' to facilitate frequent edits; see discussion of \mbox{micropayment} channels, section \ref{sec:uptime}. \section{Proof of Storage} \label{sec:storage} Storage proof transactions are periodically submitted in order to fulfill file contracts. Each storage proof targets a specific file contract. A storage proof does not need to have any inputs or outputs; only a contract ID and the proof data are required. \subsection{Algorithm} Hosts prove their storage by providing a segment of the original file and a list of hashes from the file's Merkle tree. This information is sufficient to prove that the segment came from the original file. Because proofs are submitted to the blockchain, anyone can verify their validity or invalidity. Each storage proof uses a randomly selected segment. The random seed for challenge window $W_i$ is given by: \[ H(\text{contract ID} || H(B_{i-1})) \] where $B_{i-1}$ is the block immediately prior to the beginning of $W_i$. If the host is consistently able to demonstrate possession of a random segment, then they are very likely storing the whole file. A host storing only 50\% of the file will be unable to complete approximately 50\% of the proofs. \subsection{Block Withholding Attacks} The random number generator is subject to manipulation via block withholding attacks, in which the attacker withholds blocks until they find one that will produce a favorable random number. However, the attacker has only one chance to manipulate the random number for a particular challenge. Furthermore, withholding a block to manipulate the random number will cost the attacker the block reward. If an attacker is able to mine 50\% of the blocks, then 50\% of the challenges can be manipulated. Nevertheless, the remaining 50\% are still random, so the attacker will still fail some storage proofs. Specifically, they will fail half as many as they would without the withholding attack. To protect against such attacks, clients can specify a high challenge frequency and large penalties for missing proofs. These precautions should be sufficient to deter any financially-motivated attacker that controls less than 50\% of the network's hashing power. Regardless, clients are advised to plan around potential Byzantine attacks, which may not be financially motivated. \subsection{Closed Window Attacks} Hosts can only complete a storage proof if their proof transaction makes it into the blockchain. Miners could maliciously exclude storage proofs from blocks, depriving themselves of transaction fees but forcing a penalty on hosts. Alternatively, miners could extort hosts by requiring large fees to include storage proofs, knowing that they are more important than the average transaction. This is termed a \textit{closed window attack}, because the malicious miner has artificially ``closed the window.'' The defense for this is to use a large window size. Hosts can reasonably assume that some percentage of miners will include their proofs in return for a transaction fee. Because hosts consent to all file contracts, they are free to reject any contract that they feel leaves them vulnerable to closed window attacks. \section{Arbitrary Transaction Data} Each transaction has an arbitrary data field which can be used for any type of information. Nodes will be required to store the arbitrary data if it is signed by any signature in the transaction. Nodes will initially accept up to 64 KB of arbitrary data per block. This arbitrary data provides hosts and clients with a decentralized way to organize themselves. It can be used to advertise available space or files seeking a host, or to create a decentralized file tracker. Arbitrary data could also be used to implement other types of soft forks. This would be done by creating an ``anyone-can-spend'' output but with restrictions specified in the arbitrary data. Miners that understand the restrictions can block any transaction that spends the output without satisfying the necessary stipulations. Naive nodes will stay synchronized without needing to be able to parse the arbitrary data. \section{Storage Ecosystem} Sia relies on an ecosystem that facilitates decentralized storage. Storage providers can use the arbitrary data field to announce themselves to the network. This can be done using standardized template that clients will be able to read. Clients can use these announcements to create a database of potential hosts, and form contracts with only those they trust. \subsection{Host Protections} \label{subsec:host-protect} A contract requires consent from both the storage provider and their client, allowing the provider to reject unfavorable terms or unwanted (e.g. illegal) files. The provider may also refuse to sign a contract until the entire file has been uploaded to them. Contract terms give storage providers some flexibility. They can advertise themselves as minimally reliable, offering a low price and a agreeing to minimal penalties for losing files; or they can advertise themselves as highly reliable, offering a higher price and agreeing to harsher penalties for losing files. An efficient market will optimize storage strategies. Hosts are vulnerable to denial of service attacks, which could prevent them from submitting storage proofs or transferring files. It is the responsibility of the host to protect themselves from such attacks. \subsection{Client Protections} \label{subsec:client-protect} Clients can use erasure codes, such as regenerating codes \cite{reg}, to safeguard against hosts going offline. These codes typically operate by splitting a file into $n$ pieces, such that the file can be recovered from any subset of $m$ unique pieces. (The values of $n$ and $m$ vary based on the specific erasure code and redundancy factor.) Each piece is then encrypted and stored across many hosts. This allows a client to attain high file availability even if the average network reliability is low. As an extreme example, if only 10 out of 100 pieces are needed to recover the file, then the client is actually relying on the 10 most reliable hosts, rather than the average reliability. Availability can be further improved by rehosting file pieces whose hosts have gone offline. Other metrics benefit from this strategy as well; the client can reduce latency by downloading from the closest 10 hosts, or increase download speed by downloading from the 10 fastest hosts. These downloads can be run in parallel to maximize available bandwidth. \subsection{Uptime Incentives} \label{sec:uptime} The storage proofs contain no mechanism to enforce constant uptime. There are also no provisions that require hosts to transfer files to clients upon request. One might expect, then, to see hosts holding their clients' files hostage and demanding exorbitant fees to download them. However, this attack is mitigated through the use of erasure codes, as described in section \ref{subsec:client-protect}. The strategy gives clients the freedom to ignore uncooperative hosts and work only with those that are cooperative. As a result, power shifts from the host to the client, and the ``download fee'' becomes an ``upload incentive.'' In this scenario, clients offer a reward for being sent a file, and hosts must compete to provide the best quality of service. Clients may request a file at any time, which incentivizes hosts to maximize uptime in order to collect as many rewards as possible. Clients can also incentivize greater throughput and lower latency via proportionally larger rewards. Clients could even perform random ``checkups'' that reward hosts simply for being online, even if they do not wish to download anything. However, we reiterate that uptime incentives are not part of the Sia protocol; they are entirely dependent on client behavior. Payment for downloads is expected to be offered through preexisting micropayment channels \cite{mpc}. Micropayment channels allow clients to make many consecutive small payments with minimal latency and blockchain bloat. Hosts could transfer a small segment of the file and wait to receive a micropayment before proceeding. The use of many consecutive payments allows each party to minimize the risk of being cheated. Micropayments are small enough and fast enough that payments could be made every few seconds without having any major effect on throughput. \subsection{Basic Reputation System} Clients need a reliable method for picking quality hosts. Analyzing their history is insufficient, because the history could be spoofed. A host could repeatedly form contracts with itself, agreeing to store large ``fake'' files, such as a file containing only zeros. It would be trivial to perform storage proofs on such data without actually storing anything. To mitigate this Sybil attack, clients can require that hosts that announce themselves in the arbitrary data section also include a large volume of time locked coins. If 10 coins are time locked 14 days into the future, then the host can be said to have created a lock valued at 140 coin-days. By favoring hosts that have created high-value locks, clients can mitigate the risk of Sybil attacks, as valuable locks are not trivial to create. Each client can choose their own equation for picking hosts, and can use a large number of factors, including price, lock value, volume of storage being offered, and the penalties hosts are willing to pay for losing files. More complex systems, such as those that use human review or other metrics, could be implemented out-of-band in a more centralized setting. \section{Siafunds} Sia is a product of Nebulous Incorporated. Nebulous is a for-profit company, and Sia is intended to become a primary source of income for the company. Currency premining is not a stable source of income, as it requires creating a new currency and tethering the company's revenue to the currency's increasing value. When the company needs to spend money, it must trade away portions of its source of income. Additionally, premining means that one entity has control over a large volume of the currency, and therefore potentially large and disruptive control over the market. Instead, Nebulous intends to generate revenue from Sia in a manner proportional to the value added by Sia, as determined by the value of the contracts set up between clients and hosts. This is accomplished by imposing a fee on all contracts. When a contract is created, 3.9\% of the contract fund is removed and distributed to the holders of \textit{siafunds}. Nebulous Inc. will initially hold approx. 88\% of the siafunds, and the early crowd-fund backers of Sia will hold the rest. Siafunds can be sent to other addresses, in the same way that siacoins can be sent to other addresses. They cannot, however, be used to fund contracts or miner fees. When siafunds are transferred to a new address, an additional unspent output is created, containing all of the siacoins that have been earned by the siafunds since their previous transfer. These siacoins are sent to the same address as the siafunds. \section{Economics of Sia} The primary currency of Sia is the siacoin. The supply of siacoins will increase permanently, and all fresh supply will be given to miners as a block subisdy. The first block will have 300,000 coins minted. This number will decrease by 1 coin per block, until a minimum of 30,000 coins per block is reached. Following a target of 10 minutes between blocks, the annual growth in supply is:\\ \tabcolsep=0.11cm \noindent \begin{tabular}{ l | *{7}{c} } Year & 1 & 2 & 3 & 4 & 5 & 8 & 20 \\ \hline Growth & 90\% & 39\% & 21\% & 11.5\% & 4.4\% & 3.2\% & 2.3\% \end{tabular} \\ There are inefficiencies within the Sia incentive scheme. The primary goal of Sia is to provide a blockchain that enforces storage contracts. The mining reward, however, is only indirectly linked to the total value of contracts being created. The siacoin, especially initially, is likely to have high volatility. Hosts can be adversely affected if the value of the currency shifts mid-contract. As a result, we expect to see hosts increasing the price of long-term contracts as a hedge against volatility. Additionally, hosts can advertise their prices in a more stable currency (like USD) and convert to siacoin immediately before finalizing a contract. Eventually, the use of two-way pegs with other crypto-assets will give hosts additional means to insulate themselves from volatility. % \section{Under Consideration} % The primary foundation of Sia has been established above. % Other considerations, such as mining algorithms, block time, etc., can be assumed to mirror those found in Bitcoin. % Giving careful attention to ``A Treatise on Altcoins'' \cite{alts}, we are considering the following changes to Sia for the overall improvement of the cryptocurrency. % We caution that these propositions have not yet been rigorously examined from a security standpoint. % \subsection{Flexible Contracts} % Contracts, in their present form, are fairly strict. % There is a set penalty for each missed storage proof, and a termination upon $n$ total missed storage proofs. % Increased flexibility in the penalty schedule may be desirable. % Contracts are also permanent, creating what is essentially an uneditable file on the network. % There may be value in enabling clients and hosts to negotiate an updated file hash or other updated contract terms. % Updating the terms of the contract would require consent from all parties. % \subsection{Proof of Existence Windows} % In an attempt to partially resolve the closed window attacks, we could use a proof of existence strategy. % A host can create a hash of the storage proof which they submit to the blockchain within the window. % The host then has a greatly extended window in which they can demonstrate that the proof of storage was created during the required window. % This has two advantages. % First, an attacker cannot selectively exclude proof of existence hashes, because there's no way to figure out who owns each hash. % Either the attacker doesn't include any unknown proof of existence hashes, or the attacker risks including undesired proof of existence hashes. % Second, this allows hosts to submit small transactions to the network during peak hours and then the larger transactions when the traffic has died down. % A further improvement would enable Merkle Tree proofs of existence. % This would enable a host to submit multiple proofs of storage in a single proof of existence hash. % % delete/expand this section? % \subsection{Siafund the Miner fees} % Have some portion of siafunds contribute to the miner fees, which ensures that miners have compensation so long as Sia is being used for its core purpose - storage. % \subsection{Miner Fee Adjustments} % If a block has miner fees which are significantly higher than the fees in the current block, there is incentive for miners to re-mine the previous block and change who gets the miner fees. % This can be mitigated by putting all of the fees into a pool which pays out 50\% every block, making re-mining unprofitable for any party with less than 50\% of the network hashing power. % Link to discussion threads of this potential change. % Additionally, miners have incentives not to propagate high fee transactions, because this will prevent other miners from mining the transaction and collecting the fees. % It may be possible to construct a system using fee deterioration that means a miner has the highest expected total reward when the transaction is mined as soon as possible - regardless of who mines the transaction. % Link to discussion threads of this potential change. % \subsection{More Frequent Target Adjustments} % % This section could use a few citations, but the discussion on this seems pretty well scattered. I could find things like the Kimoto Gravity Well, but other than alts.pdf I couldn't find any comments by respected individuals. I know that these discussions are out there, I've seen them before, just can't find them. % Bitcoin adjusts its block difficulty every 2016 blocks, with a target of 10 minutes per block. % This often means that the final blocks in a difficulty window are produced faster than 10 minutes (sometimes substantially) on account of rapid growth in available mining power. % To mitigate this effect, we are considering adjusting the difficulty after every block. % % I don't understand this paragraph % The bi-weekly adjustments to the Bitcoin difficulty can also cause coordinated drops in mining power---all at once, miners lose a percentage of their dollars-per-energy efficiency. % Difficulty adjustments every block creates a much smoother function for when mining rigs are no longer profitable. % The clamp on mining growth can also be increased. % The clamp serves to prevent an attacker from being easily manipulating the difficulty. % % 2,016,000 what? % In Bitcoin, raising the difficulty from 1000 to 4000 requires a minimum of 2,016,000 of work, and the difficulty can adjust by a maximum of 4x every week. % If the difficulty is being adjusted every block, however, and clamped at 1.001\% per block, an attacker will need 3,000,050 work to raise the difficulty from 1000 to 4000. % At this rate, the difficulty can shift by a maximum of 7.5x every week, which both increases the flexibility of the difficulty and makes difficulty raising attacks more difficult. % Though the difficulty will be adjusted every block, it will still be adjusted according to the amount of time taken to produce the previous 2016 blocks, preventing randomly fast or slow blocks from having large impact on the network. % \subsection{Committing to State} % One thing that could allow for substantially lighter weight clients is if the miners committed to the current state of the network, instead of just to the new transactions. % This would mean creating a structure for a database that represents the state of the network and hashing it. % We could follow suggestions similar to those presented in ``Ultimate blockchain compression'' \cite{ubc}. % \subsection{Variance Enforced Merge Mining} % Bitcoin enforces a number of leading 0s on a winning block. % Sia could enforce something like a single leading 1, followed by a bunch of leading 0s. % This creates the property that no hash is ever valid for both Bitcoin and Sia. % The value to this is that the number of payouts a miner gets from finding blocks goes up. % The total payout is still the same, but the number of payouts increases by the number of blocks that would have been valid for both. % A block that solves the coin with the highest difficulty will always be valid for both blockchains. % (I need to read more about merge mining before I publish this section) \section{Conclusion} Sia is a variant on the Bitcoin protocol that enables decentralized file storage via cryptographic contracts. These contracts can be used to enforce storage agreements between clients and hosts. After agreeing to store a file, a host must regularly submit storage proofs to the network. The host will automatically be compensated for storing the file regardless of the behavior of the client. Importantly, contracts do not require hosts to transfer files back to their client when requested. Instead, an out-of-band ecosystem must be created to reward hosts for uploading. Clients and hosts must also find a way to coordinate; one mechanism would be the arbitrary data field in the blockchain. Various precautions have been enumerated which mitigate Sybil attacks and the unreliability of hosts. Siafunds are used as a mechanism of generating revenue for Nebulous Inc., the company responsible for the release and maintenance of Sia. By using Siafunds instead of premining, Nebulous more directly correlates revenue to actual use of the network, and is largely unaffected by market games that malicious entities may play with the network currency. Miners may also derive a part of their block subsidy from siafunds, with similar benefits. Long term, we hope to add support for two-way-pegs with various currencies, which would enable consumers to insulate themselves from the instability of a single currency. We believe Sia will provide a fertile platform for decentralized cloud storage in trustless environments. \onecolumn \begin{thebibliography}{9} \bibitem{btc} Satoshi Nakamoto, \emph{Bitcoin: A Peer-to-Peer Electronic Cash System}. \bibitem{merkle} R.C. Merkle, \emph{Protocols for public key cryptosystems}, In Proc. 1980 Symposium on Security and Privacy, IEEE Computer Society, pages 122-133, April 1980. \bibitem{cpr} Hovav Shacham, Brent Waters, \emph{Compact Proofs of Retrievability}, Proc. of Asiacrypt 2008, vol. 5350, Dec 2008, pp. 90-107. \bibitem{reg} K. V. Rashmi, Nihar B. Shah, and P. Vijay Kumar, \emph{Optimal Exact-Regenerating Codes for Distributed Storage at the MSR and MBR Points via a Product-Matrix Construction}. \bibitem{side} Adam Back, Matt Corallo, Luke Dashjr, Mark Friedenbach, Gregory Maxwell, Andrew Miller, Andrew Peolstra, Jorge Timon, Pieter Wuille, \emph{Enabling Blockchain Innovations with Pegged Sidechains}. \bibitem{alts} Andrew Poelstra, \emph{A Treatise on Altcoins}. \bibitem{ibf} Gavin Andresen, \emph{O(1) Block Propagation}, https://gist.github.com/gavinandresen/e20c3b5a1d4b97f79ac2 \bibitem{hdw} Gregory Maxwell, \emph{Deterministic Wallets}, https://bitcointalk.org/index.php?topic=19137.0 \bibitem{ubc} etotheipi, Ultimate blockchain compression w/ trust-free lite nodes, \newline https://bitcointalk.org/index.php?topic=88208.0 \bibitem{poc} Gregory Maxwell, \emph{Proof of Storage to make distributed resource consumption costly.} \newline https://bitcointalk.org/index.php?topic=310323.0 \bibitem{mpc} Mike Hearn, \emph{Rapidly-adjusted (micro)payments to a pre-determined party},\newline https://en.bitcoin.it/wiki/Contracts\#Example\_7:\_Rapidly-adjusted\_.28micro.29payments\_to\_a\_pre-determined\_party \bibitem{btcdg} Bitcoin Developer Guide, https://bitcoin.org/en/developer-guide \end{thebibliography} \end{document} Sia-1.0.4/encoding/000077500000000000000000000000001302033070500140075ustar00rootroot00000000000000Sia-1.0.4/encoding/integers.go000066400000000000000000000014531302033070500161610ustar00rootroot00000000000000package encoding import ( "encoding/binary" ) // EncInt64 encodes an int64 as a slice of 8 bytes. func EncInt64(i int64) (b []byte) { b = make([]byte, 8) binary.LittleEndian.PutUint64(b, uint64(i)) return } // DecInt64 decodes a slice of 8 bytes into an int64. // If len(b) < 8, the slice is padded with zeros. func DecInt64(b []byte) int64 { b2 := make([]byte, 8) copy(b2, b) return int64(binary.LittleEndian.Uint64(b2)) } // EncUint64 encodes a uint64 as a slice of 8 bytes. func EncUint64(i uint64) (b []byte) { b = make([]byte, 8) binary.LittleEndian.PutUint64(b, i) return } // DecUint64 decodes a slice of 8 bytes into a uint64. // If len(b) < 8, the slice is padded with zeros. func DecUint64(b []byte) uint64 { b2 := make([]byte, 8) copy(b2, b) return binary.LittleEndian.Uint64(b2) } Sia-1.0.4/encoding/marshal.go000066400000000000000000000245061302033070500157740ustar00rootroot00000000000000// Package encoding converts arbitrary objects into byte slices, and vis // versa. It also contains helper functions for reading and writing length- // prefixed data. See doc/Encoding.md for the full encoding specification. package encoding import ( "bytes" "errors" "fmt" "io" "os" "reflect" ) const ( maxDecodeLen = 12e6 // 12 MB maxSliceLen = 5e6 // 5 MB ) var ( errBadPointer = errors.New("cannot decode into invalid pointer") ) type ( // GenericMarshaler marshals objects into byte slices and unmarshals byte // slices into objects. GenericMarshaler interface { Marshal(interface{}) []byte Unmarshal([]byte, interface{}) error } // A SiaMarshaler can encode and write itself to a stream. SiaMarshaler interface { MarshalSia(io.Writer) error } // A SiaUnmarshaler can read and decode itself from a stream. SiaUnmarshaler interface { UnmarshalSia(io.Reader) error } // StdGenericMarshaler is an implementation of GenericMarshaler that uses // the encoding.Marshal and encoding.Unmarshal functions to perform // its marshaling/unmarshaling. StdGenericMarshaler struct{} // An Encoder writes objects to an output stream. Encoder struct { w io.Writer } ) // Encode writes the encoding of v to the stream. For encoding details, see // the package docstring. func (e *Encoder) Encode(v interface{}) error { return e.encode(reflect.ValueOf(v)) } // EncodeAll encodes a variable number of arguments. func (e *Encoder) EncodeAll(vs ...interface{}) error { for _, v := range vs { if err := e.Encode(v); err != nil { return err } } return nil } // write catches instances where short writes do not return an error. func (e *Encoder) write(p []byte) error { n, err := e.w.Write(p) if n != len(p) && err == nil { return io.ErrShortWrite } return err } // Encode writes the encoding of val to the stream. For encoding details, see // the package docstring. func (e *Encoder) encode(val reflect.Value) error { // check for MarshalSia interface first if val.CanInterface() { if m, ok := val.Interface().(SiaMarshaler); ok { return m.MarshalSia(e.w) } } switch val.Kind() { case reflect.Ptr: // write either a 1 or 0 if err := e.Encode(!val.IsNil()); err != nil { return err } if !val.IsNil() { return e.encode(val.Elem()) } case reflect.Bool: if val.Bool() { return e.write([]byte{1}) } else { return e.write([]byte{0}) } case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return e.write(EncInt64(val.Int())) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: return e.write(EncUint64(val.Uint())) case reflect.String: return WritePrefix(e.w, []byte(val.String())) case reflect.Slice: // slices are variable length, so prepend the length and then fallthrough to array logic if err := e.write(EncUint64(uint64(val.Len()))); err != nil { return err } if val.Len() == 0 { return nil } fallthrough case reflect.Array: // special case for byte arrays if val.Type().Elem().Kind() == reflect.Uint8 { // if the array is addressable, we can optimize a bit here if val.CanAddr() { return e.write(val.Slice(0, val.Len()).Bytes()) } // otherwise we have to copy into a newly allocated slice slice := reflect.MakeSlice(reflect.SliceOf(val.Type().Elem()), val.Len(), val.Len()) reflect.Copy(slice, val) return e.write(slice.Bytes()) } // normal slices/arrays are encoded by sequentially encoding their elements for i := 0; i < val.Len(); i++ { if err := e.encode(val.Index(i)); err != nil { return err } } return nil case reflect.Struct: for i := 0; i < val.NumField(); i++ { if err := e.encode(val.Field(i)); err != nil { return err } } return nil } // Marshalling should never fail. If it panics, you're doing something wrong, // like trying to encode a map or an unexported struct field. panic("could not marshal type " + val.Type().String()) } // NewEncoder returns a new encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { return &Encoder{w} } // Marshal returns the encoding of v. For encoding details, see the package // docstring. func Marshal(v interface{}) []byte { b := new(bytes.Buffer) NewEncoder(b).Encode(v) // no error possible when using a bytes.Buffer return b.Bytes() } // MarshalAll encodes all of its inputs and returns their concatenation. func MarshalAll(vs ...interface{}) []byte { b := new(bytes.Buffer) enc := NewEncoder(b) // Error from EncodeAll is ignored as encoding cannot fail when writing // to a bytes.Buffer. _ = enc.EncodeAll(vs...) return b.Bytes() } // WriteFile writes v to a file. The file will be created if it does not exist. func WriteFile(filename string, v interface{}) error { file, err := os.Create(filename) if err != nil { return err } defer file.Close() err = NewEncoder(file).Encode(v) if err != nil { return errors.New("error while writing " + filename + ": " + err.Error()) } return nil } // A Decoder reads and decodes values from an input stream. type Decoder struct { r io.Reader n int } // Read implements the io.Reader interface. It also keeps track of the total // number of bytes decoded, and panics if that number exceeds a global // maximum. func (d *Decoder) Read(p []byte) (int, error) { n, err := d.r.Read(p) // enforce an absolute maximum size limit if d.n += n; d.n > maxDecodeLen { panic("encoded type exceeds size limit") } return n, err } // Decode reads the next encoded value from its input stream and stores it in // v, which must be a pointer. The decoding rules are the inverse of those // specified in the package docstring. func (d *Decoder) Decode(v interface{}) (err error) { // v must be a pointer pval := reflect.ValueOf(v) if pval.Kind() != reflect.Ptr || pval.IsNil() { return errBadPointer } // catch decoding panics and convert them to errors // note that this allows us to skip boundary checks during decoding defer func() { if r := recover(); r != nil { err = fmt.Errorf("could not decode type %s: %v", pval.Elem().Type().String(), r) } }() // reset the read count d.n = 0 d.decode(pval.Elem()) return } // DecodeAll decodes a variable number of arguments. func (d *Decoder) DecodeAll(vs ...interface{}) error { for _, v := range vs { if err := d.Decode(v); err != nil { return err } } return nil } // readN reads n bytes and panics if the read fails. func (d *Decoder) readN(n int) []byte { b := make([]byte, n) _, err := io.ReadFull(d, b) if err != nil { panic(err) } return b } // readPrefix reads a length-prefixed byte slice and panics if the read fails. func (d *Decoder) readPrefix() []byte { b, err := ReadPrefix(d, maxSliceLen) if err != nil { panic(err) } return b } // decode reads the next encoded value from its input stream and stores it in // val. The decoding rules are the inverse of those specified in the package // docstring. func (d *Decoder) decode(val reflect.Value) { // check for UnmarshalSia interface first if val.CanAddr() && val.Addr().CanInterface() { if u, ok := val.Addr().Interface().(SiaUnmarshaler); ok { err := u.UnmarshalSia(d) if err != nil { panic(err) } return } } switch val.Kind() { case reflect.Ptr: var valid bool d.decode(reflect.ValueOf(&valid).Elem()) // nil pointer, nothing to decode if !valid { return } // make sure we aren't decoding into nil if val.IsNil() { val.Set(reflect.New(val.Type().Elem())) } d.decode(val.Elem()) case reflect.Bool: b := d.readN(1) if b[0] > 1 { panic("boolean value was not 0 or 1") } val.SetBool(b[0] == 1) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: val.SetInt(DecInt64(d.readN(8))) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: val.SetUint(DecUint64(d.readN(8))) case reflect.String: val.SetString(string(d.readPrefix())) case reflect.Slice: // slices are variable length, but otherwise the same as arrays. // just have to allocate them first, then we can fallthrough to the array logic. sliceLen := DecUint64(d.readN(8)) // sanity-check the sliceLen, otherwise you can crash a peer by making // them allocate a massive slice if sliceLen > 1<<31-1 || sliceLen*uint64(val.Type().Elem().Size()) > maxSliceLen { panic("slice is too large") } else if sliceLen == 0 { return } val.Set(reflect.MakeSlice(val.Type(), int(sliceLen), int(sliceLen))) fallthrough case reflect.Array: // special case for byte arrays (e.g. hashes) if val.Type().Elem().Kind() == reflect.Uint8 { // convert val to a slice and read into it directly b := val.Slice(0, val.Len()) _, err := io.ReadFull(d, b.Bytes()) if err != nil { panic(err) } return } // arrays are unmarshalled by sequentially unmarshalling their elements for i := 0; i < val.Len(); i++ { d.decode(val.Index(i)) } return case reflect.Struct: for i := 0; i < val.NumField(); i++ { d.decode(val.Field(i)) } return default: panic("unknown type") } } // NewDecoder returns a new decoder that reads from r. func NewDecoder(r io.Reader) *Decoder { return &Decoder{r, 0} } // Unmarshal decodes the encoded value b and stores it in v, which must be a // pointer. The decoding rules are the inverse of those specified in the // package docstring for marshaling. func Unmarshal(b []byte, v interface{}) error { r := bytes.NewReader(b) return NewDecoder(r).Decode(v) } // UnmarshalAll decodes the encoded values in b and stores them in vs, which // must be pointers. func UnmarshalAll(b []byte, vs ...interface{}) error { dec := NewDecoder(bytes.NewReader(b)) return dec.DecodeAll(vs...) } // ReadFile reads the contents of a file and decodes them into v. func ReadFile(filename string, v interface{}) error { file, err := os.Open(filename) if err != nil { return err } defer file.Close() err = NewDecoder(file).Decode(v) if err != nil { return errors.New("error while reading " + filename + ": " + err.Error()) } return nil } // Marshal returns the encoding of v. For encoding details, see the package // docstring. func (m StdGenericMarshaler) Marshal(v interface{}) []byte { return Marshal(v) } // Unmarshal decodes the encoded value b and stores it in v, which must be a // pointer. The decoding rules are the inverse of those specified in the // package docstring for marshaling. func (m StdGenericMarshaler) Unmarshal(b []byte, v interface{}) error { return Unmarshal(b, v) } Sia-1.0.4/encoding/marshal_test.go000066400000000000000000000265561302033070500170420ustar00rootroot00000000000000package encoding import ( "bytes" "io" "os" "reflect" "strings" "testing" "github.com/NebulousLabs/Sia/build" ) // dummy types to test encoding type ( // basic test0 struct { B bool I int32 U uint16 S string } // slice/array test1 struct { Is []int32 Bs []byte Sa [3]string Ba [3]byte } // nested test2 struct { T test0 } // embedded test3 struct { test2 } // pointer test4 struct { P *test1 } // private field -- need to implement MarshalSia/UnmarshalSia test5 struct { s string } // private field with pointer receiver test6 struct { s string } ) func (t test5) MarshalSia(w io.Writer) error { return WritePrefix(w, []byte(t.s)) } func (t *test5) UnmarshalSia(r io.Reader) error { b, err := ReadPrefix(r, 256) t.s = string(b) return err } // same as above methods, but with a pointer receiver func (t *test6) MarshalSia(w io.Writer) error { return WritePrefix(w, []byte(t.s)) } func (t *test6) UnmarshalSia(r io.Reader) error { b, err := ReadPrefix(r, 256) t.s = string(b) return err } var testStructs = []interface{}{ test0{false, 65537, 256, "foo"}, test1{[]int32{1, 2, 3}, []byte("foo"), [3]string{"foo", "bar", "baz"}, [3]byte{'f', 'o', 'o'}}, test2{test0{false, 65537, 256, "foo"}}, test3{test2{test0{false, 65537, 256, "foo"}}}, test4{&test1{[]int32{1, 2, 3}, []byte("foo"), [3]string{"foo", "bar", "baz"}, [3]byte{'f', 'o', 'o'}}}, test5{"foo"}, &test6{"foo"}, } var testEncodings = [][]byte{ {0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 'f', 'o', 'o'}, {3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 'f', 'o', 'o', 3, 0, 0, 0, 0, 0, 0, 0, 'f', 'o', 'o', 3, 0, 0, 0, 0, 0, 0, 0, 'b', 'a', 'r', 3, 0, 0, 0, 0, 0, 0, 0, 'b', 'a', 'z', 'f', 'o', 'o'}, {0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 'f', 'o', 'o'}, {0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 'f', 'o', 'o'}, {1, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 'f', 'o', 'o', 3, 0, 0, 0, 0, 0, 0, 0, 'f', 'o', 'o', 3, 0, 0, 0, 0, 0, 0, 0, 'b', 'a', 'r', 3, 0, 0, 0, 0, 0, 0, 0, 'b', 'a', 'z', 'f', 'o', 'o'}, {3, 0, 0, 0, 0, 0, 0, 0, 'f', 'o', 'o'}, {3, 0, 0, 0, 0, 0, 0, 0, 'f', 'o', 'o'}, } // TestEncode tests the Encode function. func TestEncode(t *testing.T) { // use Marshal for convenience for i := range testStructs { b := Marshal(testStructs[i]) if !bytes.Equal(b, testEncodings[i]) { t.Errorf("bad encoding of testStructs[%d]: \nexp:\t%v\ngot:\t%v", i, testEncodings[i], b) } } // badWriter should fail on every encode enc := NewEncoder(new(badWriter)) for i := range testStructs { err := enc.Encode(testStructs[i]) if err != io.ErrShortWrite { t.Errorf("testStructs[%d]: expected ErrShortWrite, got %v", i, err) } } // special case, not covered by testStructs err := enc.Encode(struct{ U [3]uint16 }{[3]uint16{1, 2, 3}}) if err != io.ErrShortWrite { t.Error("expected ErrShortWrite, got", err) } // bad type defer func() { if recover() == nil { t.Error("expected panic, got nil") } }() enc.Encode(map[int]int{}) } // TestDecode tests the Decode function. func TestDecode(t *testing.T) { if testing.Short() { t.SkipNow() } // use Unmarshal for convenience var emptyStructs = []interface{}{&test0{}, &test1{}, &test2{}, &test3{}, &test4{}, &test5{}, &test6{}} for i := range testEncodings { err := Unmarshal(testEncodings[i], emptyStructs[i]) if err != nil { t.Error(err) } } // bad boolean err := Unmarshal([]byte{3}, new(bool)) if err == nil || err.Error() != "could not decode type bool: boolean value was not 0 or 1" { t.Error("expected bool error, got", err) } // non-pointer err = Unmarshal([]byte{1, 2, 3}, "foo") if err != errBadPointer { t.Error("expected errBadPointer, got", err) } // unknown type err = Unmarshal([]byte{1, 2, 3}, new(map[int]int)) if err == nil || err.Error() != "could not decode type map[int]int: unknown type" { t.Error("expected unknown type error, got", err) } // big slice (larger than maxSliceLen) err = Unmarshal(EncUint64(maxSliceLen+1), new([]byte)) if err == nil || err.Error() != "could not decode type []uint8: slice is too large" { t.Error("expected large slice error, got", err) } // massive slice (larger than MaxInt32) err = Unmarshal(EncUint64(1<<32), new([]byte)) if err == nil || err.Error() != "could not decode type []uint8: slice is too large" { t.Error("expected large slice error, got", err) } // many small slices (total larger than maxDecodeLen) bigSlice := strings.Split(strings.Repeat("0123456789abcdefghijklmnopqrstuvwxyz", (maxSliceLen/16)-1), "0") err = Unmarshal(Marshal(bigSlice), new([]string)) if err == nil || err.Error() != "could not decode type []string: encoded type exceeds size limit" { t.Error("expected size limit error, got", err) } // badReader should fail on every decode dec := NewDecoder(new(badReader)) for i := range testEncodings { err := dec.Decode(emptyStructs[i]) if err == nil { t.Error("expected error, got nil") } } // special case, not covered by testStructs err = dec.Decode(new([3]byte)) if err == nil || err.Error() != "could not decode type [3]uint8: EOF" { t.Error("expected EOF error, got", err) } } // TestMarshalUnmarshal tests the Marshal and Unmarshal functions, which are // inverses of each other. func TestMarshalUnmarshal(t *testing.T) { var emptyStructs = []interface{}{&test0{}, &test1{}, &test2{}, &test3{}, &test4{}, &test5{}, &test6{}} for i := range testStructs { b := Marshal(testStructs[i]) err := Unmarshal(b, emptyStructs[i]) if err != nil { t.Error(err) } } } // TestEncodeDecode tests the Encode and Decode functions, which are inverses // of each other. func TestEncodeDecode(t *testing.T) { var emptyStructs = []interface{}{&test0{}, &test1{}, &test2{}, &test3{}, &test4{}, &test5{}, &test6{}} b := new(bytes.Buffer) enc := NewEncoder(b) dec := NewDecoder(b) for i := range testStructs { enc.Encode(testStructs[i]) err := dec.Decode(emptyStructs[i]) if err != nil { t.Error(err) } } } // TestEncodeAll tests the EncodeAll function. func TestEncodeAll(t *testing.T) { // EncodeAll should produce the same result as individually encoding each // object exp := new(bytes.Buffer) enc := NewEncoder(exp) for i := range testStructs { enc.Encode(testStructs[i]) } b := new(bytes.Buffer) NewEncoder(b).EncodeAll(testStructs...) if !bytes.Equal(b.Bytes(), exp.Bytes()) { t.Errorf("expected %v, got %v", exp.Bytes(), b.Bytes()) } // hardcoded check exp.Reset() exp.Write([]byte{1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 'f', 'o', 'o', 1}) b.Reset() NewEncoder(b).EncodeAll(1, 2, "foo", true) if !bytes.Equal(b.Bytes(), exp.Bytes()) { t.Errorf("expected %v, got %v", exp.Bytes(), b.Bytes()) } } // TestDecodeAll tests the DecodeAll function. func TestDecodeAll(t *testing.T) { b := new(bytes.Buffer) NewEncoder(b).EncodeAll(testStructs...) var emptyStructs = []interface{}{&test0{}, &test1{}, &test2{}, &test3{}, &test4{}, &test5{}, &test6{}} err := NewDecoder(b).DecodeAll(emptyStructs...) if err != nil { t.Error(err) } empty0 := *emptyStructs[0].(*test0) if !reflect.DeepEqual(empty0, testStructs[0]) { t.Error("deep equal:", empty0, testStructs[0]) } empty6 := emptyStructs[6].(*test6) if !reflect.DeepEqual(empty6, testStructs[6]) { t.Error("deep equal:", empty6, testStructs[6]) } // hardcoded check b.Reset() b.Write([]byte{1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 'f', 'o', 'o', 1}) var ( one, two uint64 foo string tru bool ) err = NewDecoder(b).DecodeAll(&one, &two, &foo, &tru) if err != nil { t.Fatal(err) } else if one != 1 || two != 2 || foo != "foo" || tru != true { t.Error("values were not decoded correctly:", one, two, foo, tru) } } // TestMarshalAll tests the MarshalAll function. func TestMarshalAll(t *testing.T) { // MarshalAll should produce the same result as individually marshalling // each object var expected []byte for i := range testStructs { expected = append(expected, Marshal(testStructs[i])...) } b := MarshalAll(testStructs...) if !bytes.Equal(b, expected) { t.Errorf("expected %v, got %v", expected, b) } // hardcoded check exp := []byte{1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 'f', 'o', 'o', 1} b = MarshalAll(1, 2, "foo", true) if !bytes.Equal(b, exp) { t.Errorf("expected %v, got %v", exp, b) } } // TestUnmarshalAll tests the UnmarshalAll function. func TestUnmarshalAll(t *testing.T) { b := MarshalAll(testStructs...) var emptyStructs = []interface{}{&test0{}, &test1{}, &test2{}, &test3{}, &test4{}, &test5{}, &test6{}} err := UnmarshalAll(b, emptyStructs...) if err != nil { t.Error(err) } empty1 := *emptyStructs[1].(*test1) if !reflect.DeepEqual(empty1, testStructs[1]) { t.Error("deep equal:", empty1, testStructs[1]) } empty5 := *emptyStructs[5].(*test5) if !reflect.DeepEqual(empty5, testStructs[5]) { t.Error("deep equal:", empty5, testStructs[5]) } // hardcoded check b = []byte{1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 'f', 'o', 'o', 1} var ( one, two uint64 foo string tru bool ) err = UnmarshalAll(b, &one, &two, &foo, &tru) if err != nil { t.Fatal(err) } else if one != 1 || two != 2 || foo != "foo" || tru != true { t.Error("values were not decoded correctly:", one, two, foo, tru) } } // TestReadWriteFile tests the ReadFiles and WriteFile functions, which are // inverses of each other. func TestReadWriteFile(t *testing.T) { // standard os.MkdirAll(build.TempDir("encoding"), 0777) path := build.TempDir("encoding", "TestReadWriteFile") err := WriteFile(path, testStructs[3]) if err != nil { t.Fatal(err) } var obj test4 err = ReadFile(path, &obj) if err != nil { t.Error(err) } // bad paths err = WriteFile("/foo/bar", "baz") if err == nil { t.Error("expected error, got nil") } err = ReadFile("/foo/bar", nil) if err == nil { t.Error("expected error, got nil") } } // i5-4670K, 9a90f86: 33 MB/s func BenchmarkEncode(b *testing.B) { buf := new(bytes.Buffer) enc := NewEncoder(buf) for i := 0; i < b.N; i++ { buf.Reset() for i := range testStructs { err := enc.Encode(testStructs[i]) if err != nil { b.Fatal(err) } } } b.SetBytes(int64(buf.Len())) } // i5-4670K, 9a90f86: 26 MB/s func BenchmarkDecode(b *testing.B) { var emptyStructs = []interface{}{&test0{}, &test1{}, &test2{}, &test3{}, &test4{}, &test5{}, &test6{}} var numBytes int64 for i := 0; i < b.N; i++ { numBytes = 0 for i := range testEncodings { err := Unmarshal(testEncodings[i], emptyStructs[i]) if err != nil { b.Fatal(err) } numBytes += int64(len(testEncodings[i])) } } b.SetBytes(numBytes) } // i5-4670K, 2059112: 44 MB/s func BenchmarkMarshalAll(b *testing.B) { for i := 0; i < b.N; i++ { _ = MarshalAll(testStructs...) } b.SetBytes(int64(len(bytes.Join(testEncodings, nil)))) } // i5-4670K, 2059112: 36 MB/s func BenchmarkUnmarshalAll(b *testing.B) { var emptyStructs = []interface{}{&test0{}, &test1{}, &test2{}, &test3{}, &test4{}, &test5{}, &test6{}} structBytes := bytes.Join(testEncodings, nil) for i := 0; i < b.N; i++ { err := UnmarshalAll(structBytes, emptyStructs...) if err != nil { b.Fatal(err) } } b.SetBytes(int64(len(structBytes))) } Sia-1.0.4/encoding/prefix.go000066400000000000000000000025101302033070500156310ustar00rootroot00000000000000package encoding import ( "fmt" "io" ) // ReadPrefix reads an 8-byte length prefixes, followed by the number of bytes // specified in the prefix. The operation is aborted if the prefix exceeds a // specified maximum length. func ReadPrefix(r io.Reader, maxLen uint64) ([]byte, error) { prefix := make([]byte, 8) if _, err := io.ReadFull(r, prefix); err != nil { return nil, err } dataLen := DecUint64(prefix) if dataLen > maxLen { return nil, fmt.Errorf("length %d exceeds maxLen of %d", dataLen, maxLen) } // read dataLen bytes data := make([]byte, dataLen) _, err := io.ReadFull(r, data) return data, err } // ReadObject reads and decodes a length-prefixed and marshalled object. func ReadObject(r io.Reader, obj interface{}, maxLen uint64) error { data, err := ReadPrefix(r, maxLen) if err != nil { return err } return Unmarshal(data, obj) } // WritePrefix writes a length-prefixed byte slice to w. func WritePrefix(w io.Writer, data []byte) error { n, err := w.Write(EncUint64(uint64(len(data)))) if err != nil { return err } else if n != 8 { return io.ErrShortWrite } n, err = w.Write(data) if err == nil && n != len(data) { err = io.ErrShortWrite } return err } // WriteObject writes a length-prefixed object to w. func WriteObject(w io.Writer, v interface{}) error { return WritePrefix(w, Marshal(v)) } Sia-1.0.4/encoding/prefix_test.go000066400000000000000000000070211302033070500166720ustar00rootroot00000000000000package encoding import ( "bytes" "io" "testing" ) // badReader/Writer used to test error handling type badReader struct{} func (br *badReader) Read([]byte) (int, error) { return 0, io.EOF } type badWriter struct{} func (bw *badWriter) Write([]byte) (int, error) { return 0, nil } func TestReadPrefix(t *testing.T) { b := new(bytes.Buffer) // standard b.Write(append(EncUint64(3), "foo"...)) data, err := ReadPrefix(b, 3) if err != nil { t.Error(err) } else if string(data) != "foo" { t.Errorf("expected foo, got %s", data) } // 0-length b.Write(EncUint64(0)) _, err = ReadPrefix(b, 0) if err != nil { t.Error(err) } // empty b.Write([]byte{}) _, err = ReadPrefix(b, 3) if err != io.EOF { t.Error("expected EOF, got", err) } // less than 8 bytes b.Write([]byte{1, 2, 3}) _, err = ReadPrefix(b, 3) if err != io.ErrUnexpectedEOF { t.Error("expected unexpected EOF, got", err) } // exceed maxLen b.Write(EncUint64(4)) _, err = ReadPrefix(b, 3) if err == nil || err.Error() != "length 4 exceeds maxLen of 3" { t.Error("expected maxLen error, got", err) } // no data after length prefix b.Write(EncUint64(3)) _, err = ReadPrefix(b, 3) if err != io.EOF { t.Error("expected EOF, got", err) } } func TestReadObject(t *testing.T) { b := new(bytes.Buffer) var obj string // standard b.Write(EncUint64(11)) b.Write(append(EncUint64(3), "foo"...)) err := ReadObject(b, &obj, 11) if err != nil { t.Error(err) } else if obj != "foo" { t.Errorf("expected foo, got %s", obj) } // empty b.Write([]byte{}) err = ReadObject(b, &obj, 0) if err != io.EOF { t.Error("expected EOF, got", err) } // bad object b.Write(EncUint64(3)) b.WriteString("foo") // strings need an additional length prefix err = ReadObject(b, &obj, 3) if err == nil || err.Error() != "could not decode type string: "+io.ErrUnexpectedEOF.Error() { t.Error("expected unexpected EOF, got", err) } } func TestWritePrefix(t *testing.T) { b := new(bytes.Buffer) // standard err := WritePrefix(b, []byte("foo")) expected := append(EncUint64(3), "foo"...) if err != nil { t.Error(err) } else if !bytes.Equal(b.Bytes(), expected) { t.Errorf("WritePrefix wrote wrong data: expected %v, got %v", b.Bytes(), expected) } // badWriter (returns nil error, but doesn't write anything) bw := new(badWriter) err = WritePrefix(bw, []byte("foo")) if err != io.ErrShortWrite { t.Error("expected ErrShortWrite, got", err) } } func TestWriteObject(t *testing.T) { b := new(bytes.Buffer) // standard err := WriteObject(b, "foo") expected := append(EncUint64(11), append(EncUint64(3), "foo"...)...) if err != nil { t.Error(err) } else if !bytes.Equal(b.Bytes(), expected) { t.Errorf("WritePrefix wrote wrong data: expected %v, got %v", b.Bytes(), expected) } // badWriter bw := new(badWriter) err = WriteObject(bw, "foo") if err != io.ErrShortWrite { t.Error("expected ErrShortWrite, got", err) } } func TestReadWritePrefix(t *testing.T) { b := new(bytes.Buffer) // WritePrefix -> ReadPrefix data := []byte("foo") err := WritePrefix(b, data) if err != nil { t.Fatal(err) } rdata, err := ReadPrefix(b, 100) if err != nil { t.Error(err) } else if !bytes.Equal(rdata, data) { t.Errorf("read/write mismatch: wrote %s, read %s", data, rdata) } // WriteObject -> ReadObject obj := "bar" err = WriteObject(b, obj) if err != nil { t.Fatal(err) } var robj string err = ReadObject(b, &robj, 100) if err != nil { t.Error(err) } else if robj != obj { t.Errorf("read/write mismatch: wrote %s, read %s", obj, robj) } } Sia-1.0.4/modules/000077500000000000000000000000001302033070500136715ustar00rootroot00000000000000Sia-1.0.4/modules/consensus.go000066400000000000000000000242431302033070500162450ustar00rootroot00000000000000package modules import ( "errors" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/types" ) const ( // ConsensusDir is the name of the directory used for all of the consensus // persistence files. ConsensusDir = "consensus" // DiffApply indicates that a diff is being applied to the consensus set. DiffApply DiffDirection = true // DiffRevert indicates that a diff is being reverted from the consensus // set. DiffRevert DiffDirection = false ) var ( // ConsensusChangeBeginning is a special consensus change id that tells the // consensus set to provide all consensus changes starting from the very // first diff, which includes the genesis block diff. ConsensusChangeBeginning = ConsensusChangeID{} // ConsensusChangeRecent is a special consensus change id that tells the // consensus set to provide the most recent consensus change, instead of // starting from a specific value (which may not be known to the caller). ConsensusChangeRecent = ConsensusChangeID{1} // ErrBlockKnown is an error indicating that a block is already in the // database. ErrBlockKnown = errors.New("block already present in database") // ErrBlockUnsolved indicates that a block did not meet the required POW // target. ErrBlockUnsolved = errors.New("block does not meet target") // ErrInvalidConsensusChangeID indicates that ConsensusSetPersistSubscribe // was called with a consensus change id that is not recognized. Most // commonly, this means that the consensus set was deleted or replaced and // now the module attempting the subscription has desynchronized. This error // should be handled by the module, and not reported to the user. ErrInvalidConsensusChangeID = errors.New("consensus subscription has invalid id - files are inconsistent") // ErrNonExtendingBlock indicates that a block is valid but does not result // in a fork that is the heaviest known fork - the consensus set has not // changed as a result of seeing the block. ErrNonExtendingBlock = errors.New("block does not extend the longest fork") ) type ( // ConsensusChangeID is the id of a consensus change. ConsensusChangeID crypto.Hash // A DiffDirection indicates the "direction" of a diff, either applied or // reverted. A bool is used to restrict the value to these two possibilities. DiffDirection bool // A ConsensusSetSubscriber is an object that receives updates to the consensus // set every time there is a change in consensus. ConsensusSetSubscriber interface { // ProcessConsensusChange sends a consensus update to a module through // a function call. Updates will always be sent in the correct order. // There may not be any reverted blocks, but there will always be // applied blocks. ProcessConsensusChange(ConsensusChange) } // A ConsensusChange enumerates a set of changes that occurred to the consensus set. ConsensusChange struct { // ID is a unique id for the consensus change derived from the reverted // and applied blocks. ID ConsensusChangeID // RevertedBlocks is the list of blocks that were reverted by the change. // The reverted blocks were always all reverted before the applied blocks // were applied. The revered blocks are presented in the order that they // were reverted. RevertedBlocks []types.Block // AppliedBlocks is the list of blocks that were applied by the change. The // applied blocks are always all applied after all the reverted blocks were // reverted. The applied blocks are presented in the order that they were // applied. AppliedBlocks []types.Block // SiacoinOutputDiffs contains the set of siacoin diffs that were applied // to the consensus set in the recent change. The direction for the set of // diffs is 'DiffApply'. SiacoinOutputDiffs []SiacoinOutputDiff // FileContractDiffs contains the set of file contract diffs that were // applied to the consensus set in the recent change. The direction for the // set of diffs is 'DiffApply'. FileContractDiffs []FileContractDiff // SiafundOutputDiffs contains the set of siafund diffs that were applied // to the consensus set in the recent change. The direction for the set of // diffs is 'DiffApply'. SiafundOutputDiffs []SiafundOutputDiff // DelayedSiacoinOutputDiffs contains the set of delayed siacoin output // diffs that were applied to the consensus set in the recent change. DelayedSiacoinOutputDiffs []DelayedSiacoinOutputDiff // SiafundPoolDiffs are the siafund pool diffs that were applied to the // consensus set in the recent change. SiafundPoolDiffs []SiafundPoolDiff // ChildTarget defines the target of any block that would be the child // of the block most recently appended to the consensus set. ChildTarget types.Target // MinimumValidChildTimestamp defines the minimum allowed timestamp for // any block that is the child of the block most recently appended to // the consensus set. MinimumValidChildTimestamp types.Timestamp // Synced indicates whether or not the ConsensusSet is synced with its // peers. Synced bool } // A SiacoinOutputDiff indicates the addition or removal of a SiacoinOutput in // the consensus set. SiacoinOutputDiff struct { Direction DiffDirection ID types.SiacoinOutputID SiacoinOutput types.SiacoinOutput } // A FileContractDiff indicates the addition or removal of a FileContract in // the consensus set. FileContractDiff struct { Direction DiffDirection ID types.FileContractID FileContract types.FileContract } // A SiafundOutputDiff indicates the addition or removal of a SiafundOutput in // the consensus set. SiafundOutputDiff struct { Direction DiffDirection ID types.SiafundOutputID SiafundOutput types.SiafundOutput } // A DelayedSiacoinOutputDiff indicates the introduction of a siacoin output // that cannot be spent until after maturing for 144 blocks. When the output // has matured, a SiacoinOutputDiff will be provided. DelayedSiacoinOutputDiff struct { Direction DiffDirection ID types.SiacoinOutputID SiacoinOutput types.SiacoinOutput MaturityHeight types.BlockHeight } // A SiafundPoolDiff contains the value of the siafundPool before the block // was applied, and after the block was applied. When applying the diff, set // siafundPool to 'Adjusted'. When reverting the diff, set siafundPool to // 'Previous'. SiafundPoolDiff struct { Direction DiffDirection Previous types.Currency Adjusted types.Currency } // A ConsensusSet accepts blocks and builds an understanding of network // consensus. ConsensusSet interface { // AcceptBlock adds a block to consensus. An error will be returned if the // block is invalid, has been seen before, is an orphan, or doesn't // contribute to the heaviest fork known to the consensus set. If the block // does not become the head of the heaviest known fork but is otherwise // valid, it will be remembered by the consensus set but an error will // still be returned. AcceptBlock(types.Block) error // BlockAtHeight returns the block found at the input height, with a // bool to indicate whether that block exists. BlockAtHeight(types.BlockHeight) (types.Block, bool) // ChildTarget returns the target required to extend the current heaviest // fork. This function is typically used by miners looking to extend the // heaviest fork. ChildTarget(types.BlockID) (types.Target, bool) // Close will shut down the consensus set, giving the module enough time to // run any required closing routines. Close() error // ConsensusSetSubscribe adds a subscriber to the list of subscribers // and gives them every consensus change that has occurred since the // change with the provided id. There are a few special cases, // described by the ConsensusChangeX variables in this package. ConsensusSetSubscribe(ConsensusSetSubscriber, ConsensusChangeID) error // CurrentBlock returns the latest block in the heaviest known // blockchain. CurrentBlock() types.Block // Flush will cause the consensus set to finish all in-progress // routines. Flush() error // Height returns the current height of consensus. Height() types.BlockHeight // Synced returns true if the consensus set is synced with the network. Synced() bool // InCurrentPath returns true if the block id presented is found in the // current path, false otherwise. InCurrentPath(types.BlockID) bool // MinimumValidChildTimestamp returns the earliest timestamp that is // valid on the current longest fork according to the consensus set. This is // a required piece of information for the miner, who could otherwise be at // risk of mining invalid blocks. MinimumValidChildTimestamp(types.BlockID) (types.Timestamp, bool) // StorageProofSegment returns the segment to be used in the storage proof for // a given file contract. StorageProofSegment(types.FileContractID) (uint64, error) // TryTransactionSet checks whether the transaction set would be valid if // it were added in the next block. A consensus change is returned // detailing the diffs that would result from the application of the // transaction. TryTransactionSet([]types.Transaction) (ConsensusChange, error) // Unsubscribe removes a subscriber from the list of subscribers, // allowing for garbage collection and rescanning. If the subscriber is // not found in the subscriber database, no action is taken. Unsubscribe(ConsensusSetSubscriber) } ) // Append takes to ConsensusChange objects and adds all of their diffs together. // // NOTE: It is possible for diffs to overlap or be inconsistent. This function // should only be used with consecutive or disjoint consensus change objects. func (cc ConsensusChange) Append(cc2 ConsensusChange) ConsensusChange { return ConsensusChange{ RevertedBlocks: append(cc.RevertedBlocks, cc2.RevertedBlocks...), AppliedBlocks: append(cc.AppliedBlocks, cc2.AppliedBlocks...), SiacoinOutputDiffs: append(cc.SiacoinOutputDiffs, cc2.SiacoinOutputDiffs...), FileContractDiffs: append(cc.FileContractDiffs, cc2.FileContractDiffs...), SiafundOutputDiffs: append(cc.SiafundOutputDiffs, cc2.SiafundOutputDiffs...), DelayedSiacoinOutputDiffs: append(cc.DelayedSiacoinOutputDiffs, cc2.DelayedSiacoinOutputDiffs...), } } Sia-1.0.4/modules/consensus/000077500000000000000000000000001302033070500157115ustar00rootroot00000000000000Sia-1.0.4/modules/consensus/accept.go000066400000000000000000000257711302033070500175130ustar00rootroot00000000000000package consensus import ( "bytes" "errors" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) var ( errDoSBlock = errors.New("block is known to be invalid") errNoBlockMap = errors.New("block map is not in database") errInconsistentSet = errors.New("consensus set is not in a consistent state") errOrphan = errors.New("block has no known parent") ) // managedBroadcastBlock will broadcast a block to the consensus set's peers. func (cs *ConsensusSet) managedBroadcastBlock(b types.Block) { // COMPATv0.5.1 - broadcast the block to all peers <= v0.5.1 and block header to all peers > v0.5.1. var relayBlockPeers, relayHeaderPeers []modules.Peer for _, p := range cs.gateway.Peers() { if build.VersionCmp(p.Version, "0.5.1") <= 0 { relayBlockPeers = append(relayBlockPeers, p) } else { relayHeaderPeers = append(relayHeaderPeers, p) } } go cs.gateway.Broadcast("RelayBlock", b, relayBlockPeers) go cs.gateway.Broadcast("RelayHeader", b.Header(), relayHeaderPeers) } // validateHeaderAndBlock does some early, low computation verification on the // block. Callers should not assume that validation will happen in a particular // order. func (cs *ConsensusSet) validateHeaderAndBlock(tx dbTx, b types.Block) error { // Check if the block is a DoS block - a known invalid block that is expensive // to validate. id := b.ID() _, exists := cs.dosBlocks[id] if exists { return errDoSBlock } // Check if the block is already known. blockMap := tx.Bucket(BlockMap) if blockMap == nil { return errNoBlockMap } if blockMap.Get(id[:]) != nil { return modules.ErrBlockKnown } // Check for the parent. parentID := b.ParentID parentBytes := blockMap.Get(parentID[:]) if parentBytes == nil { return errOrphan } var parent processedBlock err := cs.marshaler.Unmarshal(parentBytes, &parent) if err != nil { return err } // Check that the timestamp is not too far in the past to be acceptable. minTimestamp := cs.blockRuleHelper.minimumValidChildTimestamp(blockMap, &parent) return cs.blockValidator.ValidateBlock(b, minTimestamp, parent.ChildTarget, parent.Height+1) } // checkHeaderTarget returns true if the header's ID meets the given target. func checkHeaderTarget(h types.BlockHeader, target types.Target) bool { blockHash := h.ID() return bytes.Compare(target[:], blockHash[:]) >= 0 } // validateHeader does some early, low computation verification on the header // to determine if the block should be downloaded. Callers should not assume // that validation will happen in a particular order. func (cs *ConsensusSet) validateHeader(tx dbTx, h types.BlockHeader) error { // Check if the block is a DoS block - a known invalid block that is expensive // to validate. id := h.ID() _, exists := cs.dosBlocks[id] if exists { return errDoSBlock } // Check if the block is already known. blockMap := tx.Bucket(BlockMap) if blockMap == nil { return errNoBlockMap } if blockMap.Get(id[:]) != nil { return modules.ErrBlockKnown } // Check for the parent. parentID := h.ParentID parentBytes := blockMap.Get(parentID[:]) if parentBytes == nil { return errOrphan } var parent processedBlock err := cs.marshaler.Unmarshal(parentBytes, &parent) if err != nil { return err } // Check that the target of the new block is sufficient. if !checkHeaderTarget(h, parent.ChildTarget) { return modules.ErrBlockUnsolved } // TODO: check if the block is a non extending block once headers-first // downloads are implemented. // Check that the timestamp is not too far in the past to be acceptable. minTimestamp := cs.blockRuleHelper.minimumValidChildTimestamp(blockMap, &parent) if minTimestamp > h.Timestamp { return errEarlyTimestamp } // Check if the block is in the extreme future. We make a distinction between // future and extreme future because there is an assumption that by the time // the extreme future arrives, this block will no longer be a part of the // longest fork because it will have been ignored by all of the miners. if h.Timestamp > types.CurrentTimestamp()+types.ExtremeFutureThreshold { return errExtremeFutureTimestamp } // We do not check if the header is in the near future here, because we want // to get the corresponding block as soon as possible, even if the block is in // the near future. return nil } // addBlockToTree inserts a block into the blockNode tree by adding it to its // parent's list of children. If the new blockNode is heavier than the current // node, the blockchain is forked to put the new block and its parents at the // tip. An error will be returned if block verification fails or if the block // does not extend the longest fork. // // addBlockToTree must use its own database update because it might need to // modify the database while returning an error on the block. To prevent error // tracking complexity, the error is handled inside the function so that 'nil' // can be appropriately returned by the database and the transaction can be // committed. Switching to a managed tx through bolt will make this complexity // unneeded. func (cs *ConsensusSet) addBlockToTree(b types.Block) (ce changeEntry, err error) { var nonExtending bool err = cs.db.Update(func(tx *bolt.Tx) error { pb, err := getBlockMap(tx, b.ParentID) if build.DEBUG && err != nil { panic(err) } currentNode := currentProcessedBlock(tx) newNode := cs.newChild(tx, pb, b) // modules.ErrNonExtendingBlock should be returned if the block does // not extend the current blockchain, however the changes from newChild // should be committed (which means 'nil' must be returned). A flag is // set to indicate that modules.ErrNonExtending should be returned. nonExtending = !newNode.heavierThan(currentNode) if nonExtending { return nil } var revertedBlocks, appliedBlocks []*processedBlock revertedBlocks, appliedBlocks, err = cs.forkBlockchain(tx, newNode) if err != nil { return err } for _, rn := range revertedBlocks { ce.RevertedBlocks = append(ce.RevertedBlocks, rn.Block.ID()) } for _, an := range appliedBlocks { ce.AppliedBlocks = append(ce.AppliedBlocks, an.Block.ID()) } // To have correct error handling, appendChangeLog must be called // before appending to the in-memory changelog. If this call fails, the // change is going to be reverted, but the in-memory changelog is not // going to be reverted. // // Technically, if bolt fails for some other reason (such as a // filesystem error), the in-memory changelog will be incorrect anyway. // Restarting Sia will fix it. The in-memory changelog is being phased // out. err = appendChangeLog(tx, ce) if err != nil { return err } return nil }) if err != nil { return changeEntry{}, err } if nonExtending { return changeEntry{}, modules.ErrNonExtendingBlock } return ce, nil } // managedAcceptBlock will try to add a block to the consensus set. If the // block does not extend the longest currently known chain, an error is // returned but the block is still kept in memory. If the block extends a fork // such that the fork becomes the longest currently known chain, the consensus // set will reorganize itself to recognize the new longest fork. Accepted // blocks are not relayed. // // Typically AcceptBlock should be used so that the accepted block is relayed. // This method is typically only be used when there would otherwise be multiple // consecutive calls to AcceptBlock with each successive call accepting the // child block of the previous call. func (cs *ConsensusSet) managedAcceptBlock(b types.Block) error { // Grab a lock on the consensus set. Lock is demoted later in the function, // failure to unlock before returning an error will cause a deadlock. cs.mu.Lock() // Start verification inside of a bolt View tx. err := cs.db.View(func(tx *bolt.Tx) error { // Do not accept a block if the database is inconsistent. if inconsistencyDetected(tx) { return errInconsistentSet } // Do some relatively inexpensive checks to validate the header and block. // Validation generally occurs in the order of least expensive validation // first. err := cs.validateHeaderAndBlock(boltTxWrapper{tx}, b) if err != nil { // If the block is in the near future, but too far to be acceptable, then // save the block and add it to the consensus set after it is no longer // too far in the future. // // TODO: an attacker could mine many blocks off the genesis block all in the // future and we would spawn a goroutine per each block. To fix this, either // ban peers that send lots of future blocks and stop spawning goroutines // after we are already waiting on a large number of future blocks. // // TODO: an attacker could broadcast a future block many times and we would // spawn a goroutine for each broadcast. To fix this we should create a // cache of future blocks, like we already do for DoS blocks, and only spawn // a goroutine if we haven't already spawned one for that block. To limit // the size of the cache of future blocks, make it a constant size (say 50) // over which we would evict the block furthest in the future before adding // a new block to the cache. if err == errFutureTimestamp { go func() { time.Sleep(time.Duration(b.Timestamp-(types.CurrentTimestamp()+types.FutureThreshold)) * time.Second) err := cs.managedAcceptBlock(b) if err != nil { cs.log.Debugln("WARN: failed to accept a future block:", err) } cs.managedBroadcastBlock(b) }() } return err } return nil }) if err != nil { cs.mu.Unlock() return err } // Try adding the block to the block tree. This call will perform // verification on the block before adding the block to the block tree. An // error is returned if verification fails or if the block does not extend // the longest fork. changeEntry, err := cs.addBlockToTree(b) if err != nil { cs.mu.Unlock() return err } // If appliedBlocks is 0, revertedBlocks will also be 0. if build.DEBUG && len(changeEntry.AppliedBlocks) == 0 && len(changeEntry.RevertedBlocks) != 0 { panic("appliedBlocks and revertedBlocks are mismatched!") } // Updates complete, demote the lock. cs.mu.Demote() defer cs.mu.DemotedUnlock() if len(changeEntry.AppliedBlocks) > 0 { cs.readlockUpdateSubscribers(changeEntry) } return nil } // AcceptBlock will try to add a block to the consensus set. If the block does // not extend the longest currently known chain, an error is returned but the // block is still kept in memory. If the block extends a fork such that the // fork becomes the longest currently known chain, the consensus set will // reorganize itself to recognize the new longest fork. If a block is accepted // without error, it will be relayed to all connected peers. This function // should only be called for new blocks. func (cs *ConsensusSet) AcceptBlock(b types.Block) error { err := cs.tg.Add() if err != nil { return err } defer cs.tg.Done() err = cs.managedAcceptBlock(b) if err != nil { return err } cs.managedBroadcastBlock(b) return nil } Sia-1.0.4/modules/consensus/accept_bench_test.go000066400000000000000000000107041302033070500216770ustar00rootroot00000000000000package consensus import ( "path/filepath" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/types" ) // BenchmarkAcceptEmptyBlocks measures how quckly empty blocks are integrated // into the consensus set. // // i7-4770, 1d60d69: 1.356 ms / op func BenchmarkAcceptEmptyBlocks(b *testing.B) { cst, err := createConsensusSetTester("BenchmarkEmptyBlocks") if err != nil { b.Fatal("Error creating tester: " + err.Error()) } defer cst.Close() // Create an alternate testing consensus set, which does not // have any subscribers testdir := build.TempDir(modules.ConsensusDir, "BenchmarkEmptyBlocks - 2") g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { b.Fatal(err) } cs, err := New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { b.Fatal(err) } defer cs.Close() // Synchronisze the cst and the subscriberless consensus set. h := cst.cs.dbBlockHeight() for i := types.BlockHeight(1); i <= h; i++ { id, err := cst.cs.dbGetPath(i) if err != nil { b.Fatal(err) } processedBlock, err := cst.cs.dbGetBlockMap(id) if err != nil { b.Fatal(err) } err = cs.AcceptBlock(processedBlock.Block) if err != nil { b.Fatal(err) } } b.ResetTimer() b.StopTimer() for j := 0; j < b.N; j++ { // Submit a block to the consensus set tester - which has many // subscribers. (untimed) block, err := cst.miner.AddBlock() if err != nil { b.Fatal(err) } // Submit a block to the consensus set which has no subscribers. // (timed) b.StartTimer() err = cs.AcceptBlock(block) if err != nil { b.Fatal("error accepting a block:", err) } b.StopTimer() } } // BenchmarkAcceptSmallBlocks measures how quickly smaller blocks are // integrated into the consensus set. // // i7-4770, 1d60d69: 3.579 ms / op func BenchmarkAcceptSmallBlocks(b *testing.B) { cst, err := createConsensusSetTester("BenchmarkAcceptSmallBlocks") if err != nil { b.Fatal(err) } defer cst.Close() // COMPAT v0.4.0 // // Push the height of the consensus set tester beyond the fork height. for i := 0; i < 10; i++ { _, err := cst.miner.AddBlock() if err != nil { b.Fatal(err) } } // Create an alternate testing consensus set, which does not // have any subscribers testdir := build.TempDir(modules.ConsensusDir, "BenchmarkAcceptSmallBlocks - 2") g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { b.Fatal(err) } cs, err := New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { b.Fatal("Error creating consensus: " + err.Error()) } defer cs.Close() // Synchronize the consensus set with the consensus set tester. h := cst.cs.dbBlockHeight() for i := types.BlockHeight(1); i <= h; i++ { id, err := cst.cs.dbGetPath(i) if err != nil { b.Fatal(err) } processedBlock, err := cst.cs.dbGetBlockMap(id) if err != nil { b.Fatal(err) } err = cs.AcceptBlock(processedBlock.Block) if err != nil { b.Fatal(err) } } b.ResetTimer() b.StopTimer() for j := 0; j < b.N; j++ { // Create a transaction with a miner fee, a normal siacoin output, and // a funded file contract. txnBuilder := cst.wallet.StartTransaction() err = txnBuilder.FundSiacoins(types.NewCurrency64(125e6)) if err != nil { b.Fatal(err) } // Add a small miner fee. txnBuilder.AddMinerFee(types.NewCurrency64(5e6)) // Add a siacoin output. txnBuilder.AddSiacoinOutput(types.SiacoinOutput{Value: types.NewCurrency64(20e6)}) // Add a file contract. fc := types.FileContract{ WindowStart: 1000, WindowEnd: 10005, Payout: types.NewCurrency64(100e6), ValidProofOutputs: []types.SiacoinOutput{{ Value: types.NewCurrency64(96100e3), }}, MissedProofOutputs: []types.SiacoinOutput{{ Value: types.NewCurrency64(96100e3), }}, } txnBuilder.AddFileContract(fc) txnSet, err := txnBuilder.Sign(true) if err != nil { b.Fatal(err) } // Submit the transaction set and mine the block. err = cst.tpool.AcceptTransactionSet(txnSet) if err != nil { b.Fatal(err) } block, err := cst.miner.AddBlock() if err != nil { b.Fatal(err) } // Submit the block to the consensus set without subscribers, timing // how long it takes for the block to get accepted. b.StartTimer() err = cs.AcceptBlock(block) if err != nil { b.Fatal(err) } b.StopTimer() } } Sia-1.0.4/modules/consensus/accept_reorg_test.go000066400000000000000000000234471302033070500217460ustar00rootroot00000000000000package consensus import ( "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // reorgSets contains multiple consensus sets that share a genesis block, which // can be manipulated to cause full integration blockchain reorgs. // // cstBackup is a holding place for cstMain - the blocks originally in cstMain get moved // to cstBackup so that cstMain can be reorganized without that history being lost. // Extending cstBackup will allow cstMain to be reorg'd back to its original blocks. type reorgSets struct { cstMain *consensusSetTester cstAlt *consensusSetTester cstBackup *consensusSetTester } // Close will close all of the testers in the reorgSets. Because we don't yet // have a good way to check errors on a deferred statement, a panic will be // thrown if there are any problems closing the reorgSets. func (rs *reorgSets) Close() error { err := rs.cstMain.Close() if err != nil { panic(err) } err = rs.cstAlt.Close() if err != nil { panic(err) } err = rs.cstBackup.Close() if err != nil { panic(err) } return nil } // createReorgSets creates a reorg set that is ready to be manipulated. func createReorgSets(name string) *reorgSets { cstMain, err := createConsensusSetTester(name + " - 1") if err != nil { panic(err) } cstAlt, err := createConsensusSetTester(name + " - 2") if err != nil { panic(err) } cstBackup, err := createConsensusSetTester(name + " - 3") if err != nil { panic(err) } return &reorgSets{ cstMain: cstMain, cstAlt: cstAlt, cstBackup: cstBackup, } } // save takes all of the blocks in cstMain and moves them to cstBackup. func (rs *reorgSets) save() { mainHeight := rs.cstMain.cs.dbBlockHeight() for i := types.BlockHeight(1); i <= mainHeight; i++ { id, err := rs.cstMain.cs.dbGetPath(i) if err != nil { panic(err) } pb, err := rs.cstMain.cs.dbGetBlockMap(id) if err != nil { panic(err) } // err is not checked - block may already be in cstBackup. _ = rs.cstBackup.cs.AcceptBlock(pb.Block) } // Check that cstMain and cstBackup are even. if rs.cstMain.cs.dbCurrentProcessedBlock().Block.ID() != rs.cstBackup.cs.dbCurrentProcessedBlock().Block.ID() { panic("could not save cstMain into cstBackup") } if rs.cstMain.cs.dbConsensusChecksum() != rs.cstBackup.cs.dbConsensusChecksum() { panic("reorg checksums do not match after saving") } } // extend adds blocks to cstAlt until cstAlt has more weight than cstMain. Then // cstMain is caught up, causing cstMain to perform a reorg that extends all // the way to the genesis block. func (rs *reorgSets) extend() { for rs.cstMain.cs.dbBlockHeight() >= rs.cstAlt.cs.dbBlockHeight() { _, err := rs.cstAlt.miner.AddBlock() if err != nil { panic(err) } } for i := types.BlockHeight(1); i <= rs.cstAlt.cs.dbBlockHeight(); i++ { id, err := rs.cstAlt.cs.dbGetPath(i) if err != nil { panic(err) } pb, err := rs.cstAlt.cs.dbGetBlockMap(id) if err != nil { panic(err) } _ = rs.cstMain.cs.AcceptBlock(pb.Block) } // Check that cstMain and cstAlt are even. if rs.cstMain.cs.dbCurrentProcessedBlock().Block.ID() != rs.cstAlt.cs.dbCurrentProcessedBlock().Block.ID() { panic("could not save cstMain into cstAlt") } if rs.cstMain.cs.dbConsensusChecksum() != rs.cstAlt.cs.dbConsensusChecksum() { panic("reorg checksums do not match after extending") } } // restore extends cstBackup until it is ahead of cstMain, and then adds all of // the blocks from cstBackup to cstMain, causing cstMain to reorg to the state // of cstBackup. func (rs *reorgSets) restore() { for rs.cstMain.cs.dbBlockHeight() >= rs.cstBackup.cs.dbBlockHeight() { _, err := rs.cstBackup.miner.AddBlock() if err != nil { panic(err) } } for i := types.BlockHeight(1); i <= rs.cstBackup.cs.dbBlockHeight(); i++ { id, err := rs.cstBackup.cs.dbGetPath(i) if err != nil { panic(err) } pb, err := rs.cstBackup.cs.dbGetBlockMap(id) if err != nil { panic(err) } _ = rs.cstMain.cs.AcceptBlock(pb.Block) } // Check that cstMain and cstBackup are even. if rs.cstMain.cs.dbCurrentProcessedBlock().Block.ID() != rs.cstBackup.cs.dbCurrentProcessedBlock().Block.ID() { panic("could not save cstMain into cstBackup") } if rs.cstMain.cs.dbConsensusChecksum() != rs.cstBackup.cs.dbConsensusChecksum() { panic("reorg checksums do not match after restoring") } } // fullReorg saves all of the blocks from cstMain into cstBackup, then extends // cstAlt until cstMain joins cstAlt in structure. Then cstBackup is extended // and cstMain is reorg'd back to have all of the original blocks. func (rs *reorgSets) fullReorg() { rs.save() rs.extend() rs.restore() } // TestIntegrationSimpleReorg tries to reorganize a simple block out of, and // then back into, the consensus set. func TestIntegrationSimpleReorg(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() rs := createReorgSets("TestIntegrationSimpleReorg") defer rs.Close() // Give a simple block to cstMain. rs.cstMain.testSimpleBlock() // Try to trigger consensus inconsistencies by doing a full reorg on the // simple block. rs.fullReorg() } // TestIntegrationSiacoinReorg tries to reorganize a siacoin output block out // of, and then back into, the consensus set. func TestIntegrationSiacoinReorg(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() rs := createReorgSets("TestIntegrationSiacoinReorg") defer rs.Close() // Give a siacoin block to cstMain. rs.cstMain.testSpendSiacoinsBlock() // Try to trigger consensus inconsistencies by doing a full reorg on the // simple block. rs.fullReorg() } // TestIntegrationValidStorageProofReorg tries to reorganize a valid storage // proof block out of, and then back into, the consensus set. func TestIntegrationValidStorageProofReorg(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() rs := createReorgSets("TestIntegrationValidStorageProofReorg") defer rs.Close() // Give a series of blocks containing a file contract and a valid storage // proof to cstMain. rs.cstMain.testValidStorageProofBlocks() // Try to trigger consensus inconsistencies by doing a full reorg on the // simple block. rs.fullReorg() } // TestIntegrationMissedStorageProofReorg tries to reorganize a valid storage // proof block out of, and then back into, the consensus set. func TestIntegrationMissedStorageProofReorg(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() rs := createReorgSets("TestIntegrationMissedStorageProofReorg") defer rs.Close() // Give a series of blocks containing a file contract and a valid storage // proof to cstMain. rs.cstMain.testMissedStorageProofBlocks() // Try to trigger consensus inconsistencies by doing a full reorg on the // simple block. rs.fullReorg() } // TestIntegrationFileContractRevisionReorg tries to reorganize a valid storage // proof block out of, and then back into, the consensus set. func TestIntegrationFileContractRevisionReorg(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() rs := createReorgSets("TestIntegrationFileContractRevisionReorg") defer rs.Close() // Give a series of blocks containing a file contract and a valid storage // proof to cstMain. rs.cstMain.testFileContractRevision() // Try to trigger consensus inconsistencies by doing a full reorg on the // simple block. rs.fullReorg() } // TestIntegrationComplexReorg stacks up blocks of all types into a single // blockchain that undergoes a massive reorg as a stress test to the codebase. func TestIntegrationComplexReorg(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() rs := createReorgSets("TestIntegrationComplexReorg") defer rs.Close() // Give a wide variety of block types to cstMain. for i := 0; i < 3; i++ { rs.cstMain.testBlockSuite() } // Give fewer blocks to cstAlt, while still using the same variety. for i := 0; i < 2; i++ { rs.cstAlt.testBlockSuite() } // Try to trigger consensus inconsistencies by doing a full reorg on the // simple block. rs.fullReorg() } /// All functions below this point are deprecated. /// // TestBuriedBadFork creates a block with an invalid transaction that's not on // the longest fork. The consensus set will not validate that block. Then valid // blocks are added on top of it to make it the longest fork. When it becomes // the longest fork, all the blocks should be fully validated and thrown out // because a parent is invalid. func TestBuriedBadFork(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestBuriedBadFork") if err != nil { t.Fatal(err) } defer cst.Close() pb := cst.cs.dbCurrentProcessedBlock() // Create a bad block that builds on a parent, so that it is part of not // the longest fork. badBlock := types.Block{ ParentID: pb.Block.ParentID, Timestamp: types.CurrentTimestamp(), MinerPayouts: []types.SiacoinOutput{{Value: types.CalculateCoinbase(pb.Height)}}, Transactions: []types.Transaction{{ SiacoinInputs: []types.SiacoinInput{{}}, // Will trigger an error on full verification but not partial verification. }}, } parent, err := cst.cs.dbGetBlockMap(pb.Block.ParentID) if err != nil { t.Fatal(err) } badBlock, _ = cst.miner.SolveBlock(badBlock, parent.ChildTarget) err = cst.cs.AcceptBlock(badBlock) if err != modules.ErrNonExtendingBlock { t.Fatal(err) } // Build another bock on top of the bad block that is fully valid, this // will cause a fork and full validation of the bad block, both the bad // block and this block should be thrown away. block := types.Block{ ParentID: badBlock.ID(), Timestamp: types.CurrentTimestamp(), MinerPayouts: []types.SiacoinOutput{{Value: types.CalculateCoinbase(pb.Height + 1)}}, } block, _ = cst.miner.SolveBlock(block, parent.ChildTarget) // okay because the target will not change err = cst.cs.AcceptBlock(block) if err == nil { t.Fatal("a bad block failed to cause an error") } } Sia-1.0.4/modules/consensus/accept_test.go000066400000000000000000000773441302033070500205550ustar00rootroot00000000000000package consensus import ( "bytes" "errors" "testing" "time" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( // validateBlockParamsGot stores the parameters passed to the most recent call // to mockBlockValidator.ValidateBlock. validateBlockParamsGot validateBlockParams mockValidBlock = types.Block{ Timestamp: 100, ParentID: mockParentID(), } mockInvalidBlock = types.Block{ Timestamp: 500, ParentID: mockParentID(), } // parentBlockSerialized is a mock serialized form of a processedBlock. parentBlockSerialized = []byte{3, 2, 1} parentBlockUnmarshaler = mockBlockMarshaler{ []predefinedBlockUnmarshal{ {parentBlockSerialized, mockParent(), nil}, }, } parentBlockHighTargetUnmarshaler = mockBlockMarshaler{ []predefinedBlockUnmarshal{ {parentBlockSerialized, mockParentHighTarget(), nil}, }, } parentBlockLowTargetUnmarshaler = mockBlockMarshaler{ []predefinedBlockUnmarshal{ {parentBlockSerialized, mockParentLowTarget(), nil}, }, } unmarshalFailedErr = errors.New("mock unmarshal failed") failingBlockUnmarshaler = mockBlockMarshaler{ []predefinedBlockUnmarshal{ {parentBlockSerialized, processedBlock{}, unmarshalFailedErr}, }, } serializedParentBlockMap = []blockMapPair{ {mockValidBlock.ParentID[:], parentBlockSerialized}, } ) type ( // mockDbBucket is an implementation of dbBucket for unit testing. mockDbBucket struct { values map[string][]byte } // mockDbTx is an implementation of dbTx for unit testing. It uses an // in-memory key/value store to mock a database. mockDbTx struct { buckets map[string]dbBucket } // predefinedBlockUnmarshal is a predefined response from mockBlockMarshaler. // It defines the unmarshaled processedBlock and error code that // mockBlockMarshaler should return in response to an input serialized byte // slice. predefinedBlockUnmarshal struct { serialized []byte unmarshaled processedBlock err error } // mockBlockMarshaler is an implementation of the encoding.GenericMarshaler // interface for unit testing. It allows clients to specify mappings of // serialized bytes into unmarshaled blocks. mockBlockMarshaler struct { p []predefinedBlockUnmarshal } // mockBlockRuleHelper is an implementation of the blockRuleHelper interface // for unit testing. mockBlockRuleHelper struct { minTimestamp types.Timestamp } // mockBlockValidator is an implementation of the blockValidator interface for // unit testing. mockBlockValidator struct { err error } // validateBlockParams stores the set of parameters passed to ValidateBlock. validateBlockParams struct { called bool b types.Block minTimestamp types.Timestamp target types.Target height types.BlockHeight } // blockMapPair represents a key-value pair in the mock block map. blockMapPair struct { key []byte val []byte } ) // Get returns the value associated with a given key. func (bucket mockDbBucket) Get(key []byte) []byte { return bucket.values[string(key)] } // Set adds a named value to a mockDbBucket. func (bucket mockDbBucket) Set(key []byte, value []byte) { bucket.values[string(key)] = value } // Bucket returns a mock dbBucket object associated with the given bucket name. func (db mockDbTx) Bucket(name []byte) dbBucket { return db.buckets[string(name)] } // Marshal is not implemented and panics if called. func (m mockBlockMarshaler) Marshal(interface{}) []byte { panic("not implemented") } // Unmarshal unmarshals a byte slice into an object based on a pre-defined map // of deserialized objects. func (m mockBlockMarshaler) Unmarshal(b []byte, v interface{}) error { for _, pu := range m.p { if bytes.Equal(b[:], pu.serialized[:]) { pv, ok := v.(*processedBlock) if !ok { panic("mockBlockMarshaler.Unmarshal expected v to be of type processedBlock") } *pv = pu.unmarshaled return pu.err } } panic("unmarshal failed: predefined unmarshal not found") } // AddPredefinedUnmarshal adds a predefinedBlockUnmarshal to mockBlockMarshaler. func (m *mockBlockMarshaler) AddPredefinedUnmarshal(u predefinedBlockUnmarshal) { m.p = append(m.p, u) } // minimumValidChildTimestamp returns the minimum timestamp of pb that can be // considered a valid block. func (brh mockBlockRuleHelper) minimumValidChildTimestamp(blockMap dbBucket, pb *processedBlock) types.Timestamp { return brh.minTimestamp } // ValidateBlock stores the parameters it receives and returns the mock error // defined by mockBlockValidator.err. func (bv mockBlockValidator) ValidateBlock(b types.Block, minTimestamp types.Timestamp, target types.Target, height types.BlockHeight) error { validateBlockParamsGot = validateBlockParams{true, b, minTimestamp, target, height} return bv.err } // mockParentID returns a mock BlockID value. func mockParentID() (parentID types.BlockID) { parentID[0] = 42 return parentID } // mockParent returns a mock processedBlock with its ChildTarget member // initialized to a dummy value. func mockParent() (parent processedBlock) { var mockTarget types.Target mockTarget[0] = 56 parent.ChildTarget = mockTarget return parent } // mockParent returns a mock processedBlock with its ChildTarget member // initialized to a the maximum value. func mockParentHighTarget() (parent processedBlock) { parent.ChildTarget = types.RootDepth return parent } // mockParent returns a mock processedBlock with its ChildTarget member // initialized to the minimum value. func mockParentLowTarget() (parent processedBlock) { return parent } // TestUnitValidateHeaderAndBlock runs a series of unit tests for validateHeaderAndBlock. func TestUnitValidateHeaderAndBlock(t *testing.T) { var tests = []struct { block types.Block dosBlocks map[types.BlockID]struct{} blockMapPairs []blockMapPair earliestValidTimestamp types.Timestamp marshaler mockBlockMarshaler useNilBlockMap bool validateBlockErr error errWant error msg string }{ { block: mockValidBlock, dosBlocks: make(map[types.BlockID]struct{}), useNilBlockMap: true, earliestValidTimestamp: mockValidBlock.Timestamp, marshaler: parentBlockUnmarshaler, errWant: errNoBlockMap, msg: "validateHeaderAndBlock should fail when no block map is found in the database", }, { block: mockValidBlock, // Create a dosBlocks map where mockValidBlock is marked as a bad block. dosBlocks: map[types.BlockID]struct{}{ mockValidBlock.ID(): {}, }, earliestValidTimestamp: mockValidBlock.Timestamp, marshaler: parentBlockUnmarshaler, errWant: errDoSBlock, msg: "validateHeaderAndBlock should reject known bad blocks", }, { block: mockValidBlock, dosBlocks: make(map[types.BlockID]struct{}), earliestValidTimestamp: mockValidBlock.Timestamp, marshaler: parentBlockUnmarshaler, errWant: errOrphan, msg: "validateHeaderAndBlock should reject a block if its parent block does not appear in the block database", }, { block: mockValidBlock, dosBlocks: make(map[types.BlockID]struct{}), blockMapPairs: serializedParentBlockMap, earliestValidTimestamp: mockValidBlock.Timestamp, marshaler: failingBlockUnmarshaler, errWant: unmarshalFailedErr, msg: "validateHeaderAndBlock should fail when unmarshaling the parent block fails", }, { block: mockInvalidBlock, dosBlocks: make(map[types.BlockID]struct{}), blockMapPairs: []blockMapPair{ {mockInvalidBlock.ParentID[:], parentBlockSerialized}, }, earliestValidTimestamp: mockInvalidBlock.Timestamp, marshaler: parentBlockUnmarshaler, validateBlockErr: errBadMinerPayouts, errWant: errBadMinerPayouts, msg: "validateHeaderAndBlock should reject a block if ValidateBlock returns an error for the block", }, { block: mockValidBlock, dosBlocks: make(map[types.BlockID]struct{}), blockMapPairs: serializedParentBlockMap, earliestValidTimestamp: mockValidBlock.Timestamp, marshaler: parentBlockUnmarshaler, errWant: nil, msg: "validateHeaderAndBlock should accept a valid block", }, } for _, tt := range tests { // Initialize the blockmap in the tx. bucket := mockDbBucket{map[string][]byte{}} for _, mapPair := range tt.blockMapPairs { bucket.Set(mapPair.key, mapPair.val) } dbBucketMap := map[string]dbBucket{} if tt.useNilBlockMap { dbBucketMap[string(BlockMap)] = nil } else { dbBucketMap[string(BlockMap)] = bucket } tx := mockDbTx{dbBucketMap} mockParent := mockParent() cs := ConsensusSet{ dosBlocks: tt.dosBlocks, marshaler: tt.marshaler, blockRuleHelper: mockBlockRuleHelper{ minTimestamp: tt.earliestValidTimestamp, }, blockValidator: mockBlockValidator{tt.validateBlockErr}, } // Reset the stored parameters to ValidateBlock. validateBlockParamsGot = validateBlockParams{} err := cs.validateHeaderAndBlock(tx, tt.block) if err != tt.errWant { t.Errorf("%s: expected to fail with `%v', got: `%v'", tt.msg, tt.errWant, err) } if err == nil || validateBlockParamsGot.called { if validateBlockParamsGot.b.ID() != tt.block.ID() { t.Errorf("%s: incorrect parameter passed to ValidateBlock - got: %v, want: %v", tt.msg, validateBlockParamsGot.b, tt.block) } if validateBlockParamsGot.minTimestamp != tt.earliestValidTimestamp { t.Errorf("%s: incorrect parameter passed to ValidateBlock - got: %v, want: %v", tt.msg, validateBlockParamsGot.minTimestamp, tt.earliestValidTimestamp) } if validateBlockParamsGot.target != mockParent.ChildTarget { t.Errorf("%s: incorrect parameter passed to ValidateBlock - got: %v, want: %v", tt.msg, validateBlockParamsGot.target, mockParent.ChildTarget) } } } } // TestCheckHeaderTarget probes the checkHeaderTarget function and checks that // the result matches the result of checkTarget. func TestCheckHeaderTarget(t *testing.T) { var b types.Block var h types.BlockHeader tests := []struct { target types.Target expected bool msg string }{ {types.RootDepth, true, "checkHeaderTarget failed for a low target"}, {types.Target{}, false, "checkHeaderTarget passed for a high target"}, {types.Target(h.ID()), true, "checkHeaderTarget failed for a same target"}, } for _, tt := range tests { if checkHeaderTarget(h, tt.target) != tt.expected { t.Error(tt.msg) } if checkHeaderTarget(h, tt.target) != checkTarget(b, tt.target) { t.Errorf("checkHeaderTarget and checkTarget do not match for target %v", tt.target) } } } // TestUnitValidateHeader runs a series of unit tests for validateHeader. func TestUnitValidateHeader(t *testing.T) { mockValidBlockID := mockValidBlock.ID() var tests = []struct { header types.BlockHeader dosBlocks map[types.BlockID]struct{} blockMapPairs []blockMapPair earliestValidTimestamp types.Timestamp marshaler mockBlockMarshaler useNilBlockMap bool errWant error msg string }{ // Test that known dos blocks are rejected. { header: mockValidBlock.Header(), // Create a dosBlocks map where mockValidBlock is marked as a bad block. dosBlocks: map[types.BlockID]struct{}{ mockValidBlock.ID(): {}, }, blockMapPairs: serializedParentBlockMap, earliestValidTimestamp: mockValidBlock.Timestamp, marshaler: parentBlockUnmarshaler, errWant: errDoSBlock, msg: "validateHeader should reject known bad blocks", }, // Test that blocks are rejected if a block map doesn't exist. { header: mockValidBlock.Header(), dosBlocks: make(map[types.BlockID]struct{}), blockMapPairs: serializedParentBlockMap, earliestValidTimestamp: mockValidBlock.Timestamp, marshaler: parentBlockUnmarshaler, useNilBlockMap: true, errWant: errNoBlockMap, msg: "validateHeader should fail when no block map is found in the database", }, // Test that known blocks are rejected. { header: mockValidBlock.Header(), dosBlocks: make(map[types.BlockID]struct{}), blockMapPairs: []blockMapPair{{mockValidBlockID[:], []byte{}}}, earliestValidTimestamp: mockValidBlock.Timestamp, marshaler: parentBlockUnmarshaler, errWant: modules.ErrBlockKnown, msg: "validateHeader should fail when the block has been seen before", }, // Test that blocks with unknown parents (orphans) are rejected. { header: mockValidBlock.Header(), dosBlocks: make(map[types.BlockID]struct{}), earliestValidTimestamp: mockValidBlock.Timestamp, marshaler: parentBlockUnmarshaler, errWant: errOrphan, msg: "validateHeader should reject a block if its parent block does not appear in the block database", }, // Test that blocks whose parents don't unmarshal are rejected. { header: mockValidBlock.Header(), dosBlocks: make(map[types.BlockID]struct{}), blockMapPairs: serializedParentBlockMap, earliestValidTimestamp: mockValidBlock.Timestamp, marshaler: failingBlockUnmarshaler, errWant: unmarshalFailedErr, msg: "validateHeader should fail when unmarshaling the parent block fails", }, // Test that blocks with too early of a timestamp are rejected. { header: mockValidBlock.Header(), dosBlocks: make(map[types.BlockID]struct{}), blockMapPairs: serializedParentBlockMap, earliestValidTimestamp: mockValidBlock.Timestamp + 1, marshaler: parentBlockHighTargetUnmarshaler, errWant: errEarlyTimestamp, msg: "validateHeader should fail when the header's timestamp is too early", }, // Test that headers in the extreme future are rejected. { header: types.BlockHeader{ Timestamp: types.CurrentTimestamp() + types.ExtremeFutureThreshold + 2, ParentID: mockParentID(), }, dosBlocks: make(map[types.BlockID]struct{}), blockMapPairs: serializedParentBlockMap, marshaler: parentBlockHighTargetUnmarshaler, errWant: errExtremeFutureTimestamp, msg: "validateHeader should fail when the header's timestamp is in the extreme future", }, // Test that headers in the near future are not rejected. { header: types.BlockHeader{ Timestamp: types.CurrentTimestamp() + types.FutureThreshold + 2, ParentID: mockParentID(), }, dosBlocks: make(map[types.BlockID]struct{}), blockMapPairs: serializedParentBlockMap, marshaler: parentBlockHighTargetUnmarshaler, errWant: nil, msg: "validateHeader should not reject headers whose timestamps are in the near future", }, // Test that blocks with too large of a target are rejected. { header: mockValidBlock.Header(), dosBlocks: make(map[types.BlockID]struct{}), blockMapPairs: serializedParentBlockMap, earliestValidTimestamp: mockValidBlock.Timestamp, marshaler: parentBlockLowTargetUnmarshaler, errWant: modules.ErrBlockUnsolved, msg: "validateHeader should reject blocks with an insufficiently low target", }, // Test that valid blocks are accepted. { header: mockValidBlock.Header(), dosBlocks: make(map[types.BlockID]struct{}), blockMapPairs: serializedParentBlockMap, earliestValidTimestamp: mockValidBlock.Timestamp, marshaler: parentBlockHighTargetUnmarshaler, errWant: nil, msg: "validateHeader should accept a valid block", }, } for _, tt := range tests { // Initialize the blockmap in the tx. bucket := mockDbBucket{map[string][]byte{}} for _, mapPair := range tt.blockMapPairs { bucket.Set(mapPair.key, mapPair.val) } dbBucketMap := map[string]dbBucket{} if tt.useNilBlockMap { dbBucketMap[string(BlockMap)] = nil } else { dbBucketMap[string(BlockMap)] = bucket } tx := mockDbTx{dbBucketMap} cs := ConsensusSet{ dosBlocks: tt.dosBlocks, marshaler: tt.marshaler, blockRuleHelper: mockBlockRuleHelper{ minTimestamp: tt.earliestValidTimestamp, }, } err := cs.validateHeader(tx, tt.header) if err != tt.errWant { t.Errorf("%s: expected to fail with `%v', got: `%v'", tt.msg, tt.errWant, err) } } } // TestIntegrationDoSBlockHandling checks that saved bad blocks are correctly // ignored. func TestIntegrationDoSBlockHandling(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestIntegrationDoSBlockHandling") if err != nil { t.Fatal(err) } defer cst.Close() // Mine a block that is valid except for containing a buried invalid // transaction. The transaction has more siacoin inputs than outputs. txnBuilder := cst.wallet.StartTransaction() err = txnBuilder.FundSiacoins(types.NewCurrency64(50)) if err != nil { t.Fatal(err) } txnSet, err := txnBuilder.Sign(true) // true sets the 'wholeTransaction' flag if err != nil { t.Fatal(err) } // Mine and submit the invalid block to the consensus set. The first time // around, the complaint should be about the rule-breaking transaction. block, target, err := cst.miner.BlockForWork() if err != nil { t.Fatal(err) } block.Transactions = append(block.Transactions, txnSet...) dosBlock, _ := cst.miner.SolveBlock(block, target) err = cst.cs.AcceptBlock(dosBlock) if err != errSiacoinInputOutputMismatch { t.Fatalf("expected %v, got %v", errSiacoinInputOutputMismatch, err) } // Submit the same block a second time. The complaint should be that the // block is already known to be invalid. err = cst.cs.AcceptBlock(dosBlock) if err != errDoSBlock { t.Fatalf("expected %v, got %v", errDoSBlock, err) } } // TestBlockKnownHandling submits known blocks to the consensus set. func TestBlockKnownHandling(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestBlockKnownHandling") if err != nil { t.Fatal(err) } defer cst.Close() // Get a block destined to be stale. block, target, err := cst.miner.BlockForWork() if err != nil { t.Fatal(err) } staleBlock, _ := cst.miner.SolveBlock(block, target) // Add two new blocks to the consensus set to block the stale block. block1, err := cst.miner.AddBlock() if err != nil { t.Fatal(err) } block2, err := cst.miner.AddBlock() if err != nil { t.Fatal(err) } // Submit the stale block. err = cst.cs.AcceptBlock(staleBlock) if err != nil && err != modules.ErrNonExtendingBlock { t.Fatal(err) } // Submit all the blocks again, looking for a 'stale block' error. err = cst.cs.AcceptBlock(block1) if err != modules.ErrBlockKnown { t.Fatalf("expected %v, got %v", modules.ErrBlockKnown, err) } err = cst.cs.AcceptBlock(block2) if err != modules.ErrBlockKnown { t.Fatalf("expected %v, got %v", modules.ErrBlockKnown, err) } err = cst.cs.AcceptBlock(staleBlock) if err != modules.ErrBlockKnown { t.Fatalf("expected %v, got %v", modules.ErrBlockKnown, err) } // Try submitting the genesis block. id, err := cst.cs.dbGetPath(0) if err != nil { t.Fatal(err) } genesisBlock, err := cst.cs.dbGetBlockMap(id) if err != nil { t.Fatal(err) } err = cst.cs.AcceptBlock(genesisBlock.Block) if err != modules.ErrBlockKnown { t.Fatalf("expected %v, got %v", modules.ErrBlockKnown, err) } } // TestOrphanHandling passes an orphan block to the consensus set. func TestOrphanHandling(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestOrphanHandling") if err != nil { t.Fatal(err) } defer cst.Close() // Try submitting an orphan block to the consensus set. The empty block can // be used, because looking for a parent is one of the first checks the // consensus set performs. orphan := types.Block{} err = cst.cs.AcceptBlock(orphan) if err != errOrphan { t.Fatalf("expected %v, got %v", errOrphan, err) } err = cst.cs.AcceptBlock(orphan) if err != errOrphan { t.Fatalf("expected %v, got %v", errOrphan, err) } } // TestMissedTarget submits a block that does not meet the required target. func TestMissedTarget(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestMissedTarget") if err != nil { t.Fatal(err) } defer cst.Close() // Mine a block that doesn't meet the target. block, target, err := cst.miner.BlockForWork() if err != nil { t.Fatal(err) } for checkTarget(block, target) && block.Nonce[0] != 255 { block.Nonce[0]++ } if checkTarget(block, target) { t.Fatal("unable to find a failing target") } err = cst.cs.AcceptBlock(block) if err != modules.ErrBlockUnsolved { t.Fatalf("expected %v, got %v", modules.ErrBlockUnsolved, err) } } // TestMinerPayoutHandling checks that blocks with incorrect payouts are // rejected. func TestMinerPayoutHandling(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestMinerPayoutHandling") if err != nil { t.Fatal(err) } defer cst.Close() // Create a block with the wrong miner payout structure - testing can be // light here because there is heavier testing in the 'types' package, // where the logic is defined. block, target, err := cst.miner.BlockForWork() if err != nil { t.Fatal(err) } block.MinerPayouts = append(block.MinerPayouts, types.SiacoinOutput{Value: types.NewCurrency64(1)}) solvedBlock, _ := cst.miner.SolveBlock(block, target) err = cst.cs.AcceptBlock(solvedBlock) if err != errBadMinerPayouts { t.Fatalf("expected %v, got %v", errBadMinerPayouts, err) } } // TestEarlyTimestampHandling checks that blocks too far in the past are // rejected. func TestEarlyTimestampHandling(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestEarlyTimestampHandling") if err != nil { t.Fatal(err) } defer cst.Close() minTimestamp := types.CurrentTimestamp() cst.cs.blockRuleHelper = mockBlockRuleHelper{ minTimestamp: minTimestamp, } // Submit a block with a timestamp in the past, before minTimestamp. block, target, err := cst.miner.BlockForWork() if err != nil { t.Fatal(err) } block.Timestamp = minTimestamp - 1 solvedBlock, _ := cst.miner.SolveBlock(block, target) err = cst.cs.AcceptBlock(solvedBlock) if err != errEarlyTimestamp { t.Fatalf("expected %v, got %v", errEarlyTimestamp, err) } } // testFutureTimestampHandling checks that blocks in the future (but not // extreme future) are handled correctly. func TestFutureTimestampHandling(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestFutureTimestampHandling") if err != nil { t.Fatal(err) } defer cst.Close() // Submit a block with a timestamp in the future, but not the extreme // future. block, target, err := cst.miner.BlockForWork() if err != nil { t.Fatal(err) } block.Timestamp = types.CurrentTimestamp() + 2 + types.FutureThreshold solvedBlock, _ := cst.miner.SolveBlock(block, target) err = cst.cs.AcceptBlock(solvedBlock) if err != errFutureTimestamp { t.Fatalf("expected %v, got %v", errFutureTimestamp, err) } // Poll the consensus set until the future block appears. for i := 0; i < 30; i++ { time.Sleep(time.Second * 3) _, err = cst.cs.dbGetBlockMap(solvedBlock.ID()) if err == nil { break } } _, err = cst.cs.dbGetBlockMap(solvedBlock.ID()) if err != nil { t.Errorf("Future block not added to consensus set.\nCurrent Timestamp %v\nFutureThreshold: %v\nBlock Timestamp %v\n", types.CurrentTimestamp(), types.FutureThreshold, block.Timestamp) } } // TestExtremeFutureTimestampHandling checks that blocks in the extreme future // are rejected. func TestExtremeFutureTimestampHandling(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestExtremeFutureTimestampHandling") if err != nil { t.Fatal(err) } defer cst.Close() // Submit a block with a timestamp in the extreme future. block, target, err := cst.miner.BlockForWork() if err != nil { t.Fatal(err) } block.Timestamp = types.CurrentTimestamp() + 2 + types.ExtremeFutureThreshold solvedBlock, _ := cst.miner.SolveBlock(block, target) err = cst.cs.AcceptBlock(solvedBlock) if err != errExtremeFutureTimestamp { t.Fatalf("expected %v, got %v", errFutureTimestamp, err) } } // TestBuriedBadTransaction tries submitting a block with a bad transaction // that is buried under good transactions. func TestBuriedBadTransaction(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestBuriedBadTransaction") if err != nil { t.Fatal(err) } defer cst.Close() pb := cst.cs.dbCurrentProcessedBlock() // Create a good transaction using the wallet. txnValue := types.NewCurrency64(1200) txnBuilder := cst.wallet.StartTransaction() err = txnBuilder.FundSiacoins(txnValue) if err != nil { t.Fatal(err) } txnBuilder.AddSiacoinOutput(types.SiacoinOutput{Value: txnValue}) txnSet, err := txnBuilder.Sign(true) if err != nil { t.Fatal(err) } err = cst.tpool.AcceptTransactionSet(txnSet) if err != nil { t.Fatal(err) } // Create a bad transaction badTxn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{}}, } txns := append(cst.tpool.TransactionList(), badTxn) // Create a block with a buried bad transaction. block := types.Block{ ParentID: pb.Block.ID(), Timestamp: types.CurrentTimestamp(), MinerPayouts: []types.SiacoinOutput{{Value: types.CalculateCoinbase(pb.Height + 1)}}, Transactions: txns, } block, _ = cst.miner.SolveBlock(block, pb.ChildTarget) err = cst.cs.AcceptBlock(block) if err == nil { t.Error("buried transaction didn't cause an error") } } // TestInconsistencyCheck puts the consensus set in to an inconsistent state // and makes sure that the santiy checks are triggering panics. func TestInconsistentCheck(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestInconsistentCheck") if err != nil { t.Fatal(err) } // Corrupt the consensus set by adding a new siafund output. sfo := types.SiafundOutput{ Value: types.NewCurrency64(1), } cst.cs.dbAddSiafundOutput(types.SiafundOutputID{}, sfo) // Catch a panic that should be caused by the inconsistency check after a // block is mined. defer func() { r := recover() if r == nil { t.Fatalf("inconsistency panic not triggered by corrupted database") } }() cst.miner.AddBlock() } // COMPATv0.4.0 // // This test checks that the hardfork scheduled for block 21,000 rolls through // smoothly. func TestTaxHardfork(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestTaxHardfork") if err != nil { t.Fatal(err) } defer cst.Close() // Create a file contract with a payout that is put into the blockchain // before the hardfork block but expires after the hardfork block. payout := types.NewCurrency64(400e6) outputSize := types.PostTax(cst.cs.dbBlockHeight(), payout) fc := types.FileContract{ WindowStart: cst.cs.dbBlockHeight() + 12, WindowEnd: cst.cs.dbBlockHeight() + 14, Payout: payout, ValidProofOutputs: []types.SiacoinOutput{{Value: outputSize}}, MissedProofOutputs: []types.SiacoinOutput{{Value: outputSize}}, UnlockHash: types.UnlockConditions{}.UnlockHash(), // The empty UC is anyone-can-spend } // Create and fund a transaction with a file contract. txnBuilder := cst.wallet.StartTransaction() err = txnBuilder.FundSiacoins(payout) if err != nil { t.Fatal(err) } fcIndex := txnBuilder.AddFileContract(fc) txnSet, err := txnBuilder.Sign(true) if err != nil { t.Fatal(err) } err = cst.tpool.AcceptTransactionSet(txnSet) if err != nil { t.Fatal(err) } _, err = cst.miner.AddBlock() if err != nil { t.Fatal(err) } // Check that the siafund pool was increased by the faulty float amount. siafundPool := cst.cs.dbGetSiafundPool() if !siafundPool.Equals64(15590e3) { t.Fatal("siafund pool was not increased correctly") } // Mine blocks until the hardfork is reached. for i := 0; i < 10; i++ { _, err = cst.miner.AddBlock() if err != nil { t.Fatal(err) } } // Submit a file contract revision and check that the payouts are able to // be the same. fcid := txnSet[len(txnSet)-1].FileContractID(fcIndex) fcr := types.FileContractRevision{ ParentID: fcid, UnlockConditions: types.UnlockConditions{}, NewRevisionNumber: 1, NewFileSize: 1, NewWindowStart: cst.cs.dbBlockHeight() + 2, NewWindowEnd: cst.cs.dbBlockHeight() + 4, NewValidProofOutputs: fc.ValidProofOutputs, NewMissedProofOutputs: fc.MissedProofOutputs, } txnBuilder = cst.wallet.StartTransaction() txnBuilder.AddFileContractRevision(fcr) txnSet, err = txnBuilder.Sign(true) if err != nil { t.Fatal(err) } err = cst.tpool.AcceptTransactionSet(txnSet) if err != nil { t.Fatal(err) } _, err = cst.miner.AddBlock() if err != nil { t.Fatal(err) } // Mine blocks until the revision goes through, such that the sanity checks // can be run. for i := 0; i < 6; i++ { _, err = cst.miner.AddBlock() if err != nil { t.Fatal(err) } } // Check that the siafund pool did not change after the submitted revision. siafundPool = cst.cs.dbGetSiafundPool() if !siafundPool.Equals64(15590e3) { t.Fatal("siafund pool was not increased correctly") } } // mockGatewayDoesBroadcast implements modules.Gateway to mock the Broadcast // method. type mockGatewayDoesBroadcast struct { modules.Gateway broadcastCalled chan struct{} } // Broadcast is a mock implementation of modules.Gateway.Broadcast that // sends a sentinel value down a channel to signal it's been called. func (g *mockGatewayDoesBroadcast) Broadcast(name string, obj interface{}, peers []modules.Peer) { g.Gateway.Broadcast(name, obj, peers) g.broadcastCalled <- struct{}{} } // TestAcceptBlockBroadcasts tests that AcceptBlock broadcasts valid blocks and // that managedAcceptBlock does not. func TestAcceptBlockBroadcasts(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := blankConsensusSetTester("TestAcceptBlockBroadcasts") if err != nil { t.Fatal(err) } defer cst.Close() mg := &mockGatewayDoesBroadcast{ Gateway: cst.cs.gateway, broadcastCalled: make(chan struct{}), } cst.cs.gateway = mg // Test that Broadcast is called for valid blocks. b, _ := cst.miner.FindBlock() err = cst.cs.AcceptBlock(b) if err != nil { t.Fatal(err) } select { case <-mg.broadcastCalled: // Broadcast is called twice, once to broadcast blocks to peers <= v0.5.1 // and once to broadcast block headers to peers > v0.5.1. <-mg.broadcastCalled case <-time.After(10 * time.Millisecond): t.Error("expected AcceptBlock to broadcast a valid block") } // Test that Broadcast is not called for invalid blocks. err = cst.cs.AcceptBlock(types.Block{}) if err == nil { t.Fatal("expected AcceptBlock to error on an invalid block") } select { case <-mg.broadcastCalled: t.Error("AcceptBlock broadcasted an invalid block") case <-time.After(10 * time.Millisecond): } // Test that Broadcast is not called in managedAcceptBlock. b, _ = cst.miner.FindBlock() err = cst.cs.managedAcceptBlock(b) if err != nil { t.Fatal(err) } select { case <-mg.broadcastCalled: t.Errorf("managedAcceptBlock should not broadcast blocks") case <-time.After(10 * time.Millisecond): } } Sia-1.0.4/modules/consensus/accept_txntypes_test.go000066400000000000000000001115011302033070500225130ustar00rootroot00000000000000package consensus import ( "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/types" ) // testBlockSuite tests a wide variety of blocks. func (cst *consensusSetTester) testBlockSuite() { cst.testSimpleBlock() cst.testSpendSiacoinsBlock() cst.testValidStorageProofBlocks() cst.testMissedStorageProofBlocks() cst.testFileContractRevision() cst.testSpendSiafunds() } // testSimpleBlock mines a simple block (no transactions except those // automatically added by the miner) and adds it to the consnesus set. func (cst *consensusSetTester) testSimpleBlock() { // Get the starting hash of the consenesus set. initialChecksum := cst.cs.dbConsensusChecksum() initialHeight := cst.cs.dbBlockHeight() initialBlockID := cst.cs.dbCurrentBlockID() // Mine and submit a block block, err := cst.miner.AddBlock() if err != nil { panic(err) } // Check that the consensus info functions changed as expected. resultingChecksum := cst.cs.dbConsensusChecksum() if initialChecksum == resultingChecksum { panic("checksum is unchanged after mining a block") } resultingHeight := cst.cs.dbBlockHeight() if resultingHeight != initialHeight+1 { panic("height of consensus set did not increase as expected") } currentPB := cst.cs.dbCurrentProcessedBlock() if currentPB.Block.ParentID != initialBlockID { panic("new processed block does not have correct information") } if currentPB.Block.ID() != block.ID() { panic("the state's current block is not reporting as the recently mined block.") } if currentPB.Height != initialHeight+1 { panic("the processed block is not reporting the correct height") } pathID, err := cst.cs.dbGetPath(currentPB.Height) if err != nil { panic(err) } if pathID != block.ID() { panic("current path does not point to the correct block") } // Revert the block that was just added to the consensus set and check for // parity with the original state of consensus. parent, err := cst.cs.dbGetBlockMap(currentPB.Block.ParentID) if err != nil { panic(err) } _, _, err = cst.cs.dbForkBlockchain(parent) if err != nil { panic(err) } if cst.cs.dbConsensusChecksum() != initialChecksum { panic("adding and reverting a block changed the consensus set") } // Re-add the block and check for parity with the first time it was added. // This test is useful because a different codepath is followed if the // diffs have already been generated. _, _, err = cst.cs.dbForkBlockchain(currentPB) if err != nil { panic(err) } if cst.cs.dbConsensusChecksum() != resultingChecksum { panic("adding, reverting, and reading a block was inconsistent with just adding the block") } } // TestIntegrationSimpleBlock creates a consensus set tester and uses it to // call testSimpleBlock. func TestIntegrationSimpleBlock(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestIntegrationSimpleBlock") if err != nil { t.Fatal(err) } defer cst.Close() cst.testSimpleBlock() } // testSpendSiacoinsBlock mines a block with a transaction spending siacoins // and adds it to the consensus set. func (cst *consensusSetTester) testSpendSiacoinsBlock() { // Create a random destination address for the output in the transaction. destAddr := randAddress() // Create a block containing a transaction with a valid siacoin output. txnValue := types.NewCurrency64(1200) txnBuilder := cst.wallet.StartTransaction() err := txnBuilder.FundSiacoins(txnValue) if err != nil { panic(err) } outputIndex := txnBuilder.AddSiacoinOutput(types.SiacoinOutput{Value: txnValue, UnlockHash: destAddr}) txnSet, err := txnBuilder.Sign(true) if err != nil { panic(err) } err = cst.tpool.AcceptTransactionSet(txnSet) if err != nil { panic(err) } // Mine and apply the block to the consensus set. _, err = cst.miner.AddBlock() if err != nil { panic(err) } // See that the destination output was created. outputID := txnSet[len(txnSet)-1].SiacoinOutputID(outputIndex) sco, err := cst.cs.dbGetSiacoinOutput(outputID) if err != nil { panic(err) } if !sco.Value.Equals(txnValue) { panic("output added with wrong value") } if sco.UnlockHash != destAddr { panic("output sent to the wrong address") } } // TestIntegrationSpendSiacoinsBlock creates a consensus set tester and uses it // to call testSpendSiacoinsBlock. func TestIntegrationSpendSiacoinsBlock(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestSpendSiacoinsBlock") if err != nil { t.Fatal(err) } defer cst.Close() cst.testSpendSiacoinsBlock() } // testValidStorageProofBlocks adds a block with a file contract, and then // submits a storage proof for that file contract. func (cst *consensusSetTester) testValidStorageProofBlocks() { // COMPATv0.4.0 - Step the block height up past the hardfork amount. This // code stops nondeterministic failures when producing storage proofs that // is related to buggy old code. for cst.cs.dbBlockHeight() <= 10 { _, err := cst.miner.AddBlock() if err != nil { panic(err) } } // Create a file (as a bytes.Buffer) that will be used for the file // contract. filesize := uint64(4e3) file, err := crypto.RandBytes(int(filesize)) if err != nil { panic(err) } merkleRoot := crypto.MerkleRoot(file) // Create a file contract that will be successful. validProofDest := randAddress() payout := types.NewCurrency64(400e6) fc := types.FileContract{ FileSize: filesize, FileMerkleRoot: merkleRoot, WindowStart: cst.cs.dbBlockHeight() + 1, WindowEnd: cst.cs.dbBlockHeight() + 2, Payout: payout, ValidProofOutputs: []types.SiacoinOutput{{ UnlockHash: validProofDest, Value: types.PostTax(cst.cs.dbBlockHeight(), payout), }}, MissedProofOutputs: []types.SiacoinOutput{{ UnlockHash: types.UnlockHash{}, Value: types.PostTax(cst.cs.dbBlockHeight(), payout), }}, } // Submit a transaction with the file contract. oldSiafundPool := cst.cs.dbGetSiafundPool() txnBuilder := cst.wallet.StartTransaction() err = txnBuilder.FundSiacoins(payout) if err != nil { panic(err) } fcIndex := txnBuilder.AddFileContract(fc) txnSet, err := txnBuilder.Sign(true) if err != nil { panic(err) } err = cst.tpool.AcceptTransactionSet(txnSet) if err != nil { panic(err) } _, err = cst.miner.AddBlock() if err != nil { panic(err) } // Check that the siafund pool was increased by the tax on the payout. siafundPool := cst.cs.dbGetSiafundPool() if !siafundPool.Equals(oldSiafundPool.Add(types.Tax(cst.cs.dbBlockHeight()-1, payout))) { panic("siafund pool was not increased correctly") } // Check that the file contract made it into the database. ti := len(txnSet) - 1 fcid := txnSet[ti].FileContractID(fcIndex) _, err = cst.cs.dbGetFileContract(fcid) if err != nil { panic(err) } // Create and submit a storage proof for the file contract. segmentIndex, err := cst.cs.StorageProofSegment(fcid) if err != nil { panic(err) } segment, hashSet := crypto.MerkleProof(file, segmentIndex) sp := types.StorageProof{ ParentID: fcid, HashSet: hashSet, } copy(sp.Segment[:], segment) txnBuilder = cst.wallet.StartTransaction() txnBuilder.AddStorageProof(sp) txnSet, err = txnBuilder.Sign(true) if err != nil { panic(err) } err = cst.tpool.AcceptTransactionSet(txnSet) if err != nil { panic(err) } _, err = cst.miner.AddBlock() if err != nil { panic(err) } // Check that the file contract has been removed. _, err = cst.cs.dbGetFileContract(fcid) if err != errNilItem { panic("file contract should not exist in the database") } // Check that the siafund pool has not changed. postProofPool := cst.cs.dbGetSiafundPool() if !postProofPool.Equals(siafundPool) { panic("siafund pool should not change after submitting a storage proof") } // Check that a delayed output was created for the valid proof. spoid := fcid.StorageProofOutputID(types.ProofValid, 0) dsco, err := cst.cs.dbGetDSCO(cst.cs.dbBlockHeight()+types.MaturityDelay, spoid) if err != nil { panic(err) } if dsco.UnlockHash != fc.ValidProofOutputs[0].UnlockHash { panic("wrong unlock hash in dsco") } if !dsco.Value.Equals(fc.ValidProofOutputs[0].Value) { panic("wrong sco value in dsco") } } // TestIntegrationValidStorageProofBlocks creates a consensus set tester and // uses it to call testValidStorageProofBlocks. func TestIntegrationValidStorageProofBlocks(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestIntegrationValidStorageProofBlocks") if err != nil { t.Fatal(err) } defer cst.Close() cst.testValidStorageProofBlocks() } // testMissedStorageProofBlocks adds a block with a file contract, and then // fails to submit a storage proof before expiration. func (cst *consensusSetTester) testMissedStorageProofBlocks() { // Create a file contract that will be successful. filesize := uint64(4e3) payout := types.NewCurrency64(400e6) missedProofDest := randAddress() fc := types.FileContract{ FileSize: filesize, FileMerkleRoot: crypto.Hash{}, WindowStart: cst.cs.dbBlockHeight() + 1, WindowEnd: cst.cs.dbBlockHeight() + 2, Payout: payout, ValidProofOutputs: []types.SiacoinOutput{{ UnlockHash: types.UnlockHash{}, Value: types.PostTax(cst.cs.dbBlockHeight(), payout), }}, MissedProofOutputs: []types.SiacoinOutput{{ UnlockHash: missedProofDest, Value: types.PostTax(cst.cs.dbBlockHeight(), payout), }}, } // Submit a transaction with the file contract. oldSiafundPool := cst.cs.dbGetSiafundPool() txnBuilder := cst.wallet.StartTransaction() err := txnBuilder.FundSiacoins(payout) if err != nil { panic(err) } fcIndex := txnBuilder.AddFileContract(fc) txnSet, err := txnBuilder.Sign(true) if err != nil { panic(err) } err = cst.tpool.AcceptTransactionSet(txnSet) if err != nil { panic(err) } _, err = cst.miner.AddBlock() if err != nil { panic(err) } // Check that the siafund pool was increased by the tax on the payout. siafundPool := cst.cs.dbGetSiafundPool() if !siafundPool.Equals(oldSiafundPool.Add(types.Tax(cst.cs.dbBlockHeight()-1, payout))) { panic("siafund pool was not increased correctly") } // Check that the file contract made it into the database. ti := len(txnSet) - 1 fcid := txnSet[ti].FileContractID(fcIndex) _, err = cst.cs.dbGetFileContract(fcid) if err != nil { panic(err) } // Mine a block to close the storage proof window. _, err = cst.miner.AddBlock() if err != nil { panic(err) } // Check that the file contract has been removed. _, err = cst.cs.dbGetFileContract(fcid) if err != errNilItem { panic("file contract should not exist in the database") } // Check that the siafund pool has not changed. postProofPool := cst.cs.dbGetSiafundPool() if !postProofPool.Equals(siafundPool) { panic("siafund pool should not change after submitting a storage proof") } // Check that a delayed output was created for the missed proof. spoid := fcid.StorageProofOutputID(types.ProofMissed, 0) dsco, err := cst.cs.dbGetDSCO(cst.cs.dbBlockHeight()+types.MaturityDelay, spoid) if err != nil { panic(err) } if dsco.UnlockHash != fc.MissedProofOutputs[0].UnlockHash { panic("wrong unlock hash in dsco") } if !dsco.Value.Equals(fc.MissedProofOutputs[0].Value) { panic("wrong sco value in dsco") } } // TestIntegrationMissedStorageProofBlocks creates a consensus set tester and // uses it to call testMissedStorageProofBlocks. func TestIntegrationMissedStorageProofBlocks(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestIntegrationMissedStorageProofBlocks") if err != nil { t.Fatal(err) } defer cst.Close() cst.testMissedStorageProofBlocks() } // testFileContractRevision creates and revises a file contract on the // blockchain. func (cst *consensusSetTester) testFileContractRevision() { // COMPATv0.4.0 - Step the block height up past the hardfork amount. This // code stops nondeterministic failures when producing storage proofs that // is related to buggy old code. for cst.cs.dbBlockHeight() <= 10 { _, err := cst.miner.AddBlock() if err != nil { panic(err) } } // Create a file (as a bytes.Buffer) that will be used for the file // contract. filesize := uint64(4e3) file, err := crypto.RandBytes(int(filesize)) if err != nil { panic(err) } merkleRoot := crypto.MerkleRoot(file) // Create a spendable unlock hash for the file contract. sk, pk, err := crypto.GenerateKeyPair() if err != nil { panic(err) } uc := types.UnlockConditions{ PublicKeys: []types.SiaPublicKey{{ Algorithm: types.SignatureEd25519, Key: pk[:], }}, SignaturesRequired: 1, } // Create a file contract that will be revised. validProofDest := randAddress() payout := types.NewCurrency64(400e6) fc := types.FileContract{ FileSize: filesize, FileMerkleRoot: crypto.Hash{}, WindowStart: cst.cs.dbBlockHeight() + 2, WindowEnd: cst.cs.dbBlockHeight() + 3, Payout: payout, ValidProofOutputs: []types.SiacoinOutput{{ UnlockHash: validProofDest, Value: types.PostTax(cst.cs.dbBlockHeight(), payout), }}, MissedProofOutputs: []types.SiacoinOutput{{ UnlockHash: types.UnlockHash{}, Value: types.PostTax(cst.cs.dbBlockHeight(), payout), }}, UnlockHash: uc.UnlockHash(), } // Submit a transaction with the file contract. txnBuilder := cst.wallet.StartTransaction() err = txnBuilder.FundSiacoins(payout) if err != nil { panic(err) } fcIndex := txnBuilder.AddFileContract(fc) txnSet, err := txnBuilder.Sign(true) if err != nil { panic(err) } err = cst.tpool.AcceptTransactionSet(txnSet) if err != nil { panic(err) } _, err = cst.miner.AddBlock() if err != nil { panic(err) } // Submit a revision for the file contract. ti := len(txnSet) - 1 fcid := txnSet[ti].FileContractID(fcIndex) fcr := types.FileContractRevision{ ParentID: fcid, UnlockConditions: uc, NewRevisionNumber: 69292, NewFileSize: filesize, NewFileMerkleRoot: merkleRoot, NewWindowStart: cst.cs.dbBlockHeight() + 1, NewWindowEnd: cst.cs.dbBlockHeight() + 2, NewValidProofOutputs: fc.ValidProofOutputs, NewMissedProofOutputs: fc.MissedProofOutputs, NewUnlockHash: uc.UnlockHash(), } ts := types.TransactionSignature{ ParentID: crypto.Hash(fcid), CoveredFields: types.CoveredFields{WholeTransaction: true}, PublicKeyIndex: 0, } txn := types.Transaction{ FileContractRevisions: []types.FileContractRevision{fcr}, TransactionSignatures: []types.TransactionSignature{ts}, } encodedSig, err := crypto.SignHash(txn.SigHash(0), sk) if err != nil { panic(err) } txn.TransactionSignatures[0].Signature = encodedSig[:] err = cst.tpool.AcceptTransactionSet([]types.Transaction{txn}) if err != nil { panic(err) } _, err = cst.miner.AddBlock() if err != nil { panic(err) } // Create and submit a storage proof for the file contract. segmentIndex, err := cst.cs.StorageProofSegment(fcid) if err != nil { panic(err) } segment, hashSet := crypto.MerkleProof(file, segmentIndex) sp := types.StorageProof{ ParentID: fcid, HashSet: hashSet, } copy(sp.Segment[:], segment) txnBuilder = cst.wallet.StartTransaction() txnBuilder.AddStorageProof(sp) txnSet, err = txnBuilder.Sign(true) if err != nil { panic(err) } err = cst.tpool.AcceptTransactionSet(txnSet) if err != nil { panic(err) } _, err = cst.miner.AddBlock() if err != nil { panic(err) } // Check that the file contract has been removed. _, err = cst.cs.dbGetFileContract(fcid) if err != errNilItem { panic("file contract should not exist in the database") } } // TestIntegrationFileContractRevision creates a consensus set tester and uses // it to call testFileContractRevision. func TestIntegrationFileContractRevision(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestIntegrationFileContractRevision") if err != nil { t.Fatal(err) } defer cst.Close() cst.testFileContractRevision() } // testSpendSiafunds spends siafunds on the blockchain. func (cst *consensusSetTester) testSpendSiafunds() { // Create a random destination address for the output in the transaction. destAddr := randAddress() // Create a block containing a transaction with a valid siafund output. txnValue := types.NewCurrency64(3) txnBuilder := cst.wallet.StartTransaction() err := txnBuilder.FundSiafunds(txnValue) if err != nil { panic(err) } outputIndex := txnBuilder.AddSiafundOutput(types.SiafundOutput{Value: txnValue, UnlockHash: destAddr}) txnSet, err := txnBuilder.Sign(true) if err != nil { panic(err) } err = cst.tpool.AcceptTransactionSet(txnSet) if err != nil { panic(err) } // Find the siafund inputs used in the txn set. var claimValues []types.Currency var claimIDs []types.SiacoinOutputID for _, txn := range txnSet { for _, sfi := range txn.SiafundInputs { sfo, err := cst.cs.dbGetSiafundOutput(sfi.ParentID) if err != nil { // It's not in the database because it's in an earlier // transaction: disregard it - testing the first layer of // dependencies is sufficient. continue } poolDiff := cst.cs.dbGetSiafundPool().Sub(sfo.ClaimStart) value := poolDiff.Div(types.SiafundCount).Mul(sfo.Value) claimValues = append(claimValues, value) claimIDs = append(claimIDs, sfi.ParentID.SiaClaimOutputID()) } } if len(claimValues) == 0 { panic("no siafund outputs created?") } // Mine and apply the block to the consensus set. _, err = cst.miner.AddBlock() if err != nil { panic(err) } // See that the destination output was created. outputID := txnSet[len(txnSet)-1].SiafundOutputID(outputIndex) sfo, err := cst.cs.dbGetSiafundOutput(outputID) if err != nil { panic(err) } if !sfo.Value.Equals(txnValue) { panic("output added with wrong value") } if sfo.UnlockHash != destAddr { panic("output sent to the wrong address") } if !sfo.ClaimStart.Equals(cst.cs.dbGetSiafundPool()) { panic("ClaimStart is not being set correctly") } // Verify that all expected claims were created and added to the set of // delayed siacoin outputs. for i, id := range claimIDs { dsco, err := cst.cs.dbGetDSCO(cst.cs.dbBlockHeight()+types.MaturityDelay, id) if err != nil { panic(err) } if !dsco.Value.Equals(claimValues[i]) { panic("expected a different claim value on the siaclaim") } } } // TestIntegrationSpendSiafunds creates a consensus set tester and uses it // to call testSpendSiafunds. func (cst *consensusSetTester) TestIntegrationSpendSiafunds(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestIntegtrationSpendSiafunds") if err != nil { t.Fatal(err) } defer cst.Close() cst.testSpendSiafunds() } // testDelayedOutputMaturity adds blocks that result in many delayed outputs // maturing at the same time, verifying that bulk maturity is handled // correctly. // TestRegressionDelayedOutputMaturity creates a consensus set tester and uses // it to call testDelayedOutputMaturity. In the past, bolt's ForEach function // had been used incorrectly resulting in the incorrect processing of bulk // delayed outputs. // testFileContractMaturity adds blocks that result in many file contracts // being closed at the same time. // TestRegressionFileContractMaturity creates a consensus set tester and uses // it to call testFileContractMaturity. In the past, bolt's ForEach function // had been used incorrectly, resulting in the incorrect processing of bulk // file contracts. /* // testPaymentChannelBlocks submits blocks to set up, use, and close a payment // channel. func (cst *consensusSetTester) testPaymentChannelBlocks() error { // The current method of doing payment channels is gimped because public // keys do not have timelocks. We will be hardforking to include timelocks // in public keys in 0.4.0, but in the meantime we need an alternate // method. // Gimped payment channels: 2-of-2 multisig where one key is controlled by // the funding entity, and one key is controlled by the receiving entity. An // address is created containing both keys, and then the funding entity // creates, but does not sign, a transaction sending coins to the channel // address. A second transaction is created that sends all the coins in the // funding output back to the funding entity. The receiving entity signs the // transaction with a timelocked signature. The funding entity will get the // refund after T blocks as long as the output is not double spent. The // funding entity then signs the first transaction and opens the channel. // // Creating the channel: // 1. Create a 2-of-2 unlock conditions, one key held by each entity. // 2. Funding entity creates, but does not sign, a transaction sending // money to the payment channel address. (txn A) // 3. Funding entity creates and signs a transaction spending the output // created in txn A that sends all the money back as a refund. (txn B) // 4. Receiving entity signs txn B with a timelocked signature, so that the // funding entity cannot get the refund for several days. The funding entity // is given a fully signed and eventually-spendable txn B. // 5. The funding entity signs and broadcasts txn A. // // Using the channel: // Each the receiving entity and the funding entity keeps a record of how // much has been sent down the unclosed channel, and watches the // blockchain for a channel closing transaction. To send more money down // the channel, the funding entity creates and signs a transaction sending // X+y coins to the receiving entity from the channel address. The // transaction is sent to the receiving entity, who will keep it and // potentially sign and broadcast it later. The funding entity will only // send money down the channel if 'work' or some other sort of event has // completed that indicates the receiving entity should get more money. // // Closing the channel: // The receiving entity will sign the transaction that pays them the most // money and then broadcast that transaction. This will spend the output // and close the channel, invalidating txn B and preventing any future // transactions from being made over the channel. The channel must be // closed before the timelock expires on the second signature in txn B, // otherwise the funding entity will be able to get a full refund. // // The funding entity should be waiting until either the receiving entity // closes the channel or the timelock expires. If the receiving entity // closes the channel, all is good. If not, then the funding entity can // close the channel and get a full refund. // Create a 2-of-2 unlock conditions, 1 key for each the sender and the // receiver in the payment channel. sk1, pk1, err := crypto.StdKeyGen.Generate() // Funding entity. if err != nil { return err } sk2, pk2, err := crypto.StdKeyGen.Generate() // Receiving entity. if err != nil { return err } uc := types.UnlockConditions{ PublicKeys: []types.SiaPublicKey{ { Algorithm: types.SignatureEd25519, Key: pk1[:], }, { Algorithm: types.SignatureEd25519, Key: pk2[:], }, }, SignaturesRequired: 2, } channelAddress := uc.UnlockHash() // Funding entity creates but does not sign a transaction that funds the // channel address. Because the wallet is not very flexible, the channel // txn needs to be fully custom. To get a custom txn, manually create an // address and then use the wallet to fund that address. channelSize := types.NewCurrency64(10e3) channelFundingSK, channelFundingPK, err := crypto.StdKeyGen.Generate() if err != nil { return err } channelFundingUC := types.UnlockConditions{ PublicKeys: []types.SiaPublicKey{{ Algorithm: types.SignatureEd25519, Key: channelFundingPK[:], }}, SignaturesRequired: 1, } channelFundingAddr := channelFundingUC.UnlockHash() fundTxnBuilder := cst.wallet.StartTransaction() if err != nil { return err } err = fundTxnBuilder.FundSiacoins(channelSize) if err != nil { return err } scoFundIndex := fundTxnBuilder.AddSiacoinOutput(types.SiacoinOutput{Value: channelSize, UnlockHash: channelFundingAddr}) fundTxnSet, err := fundTxnBuilder.Sign(true) if err != nil { return err } fundOutputID := fundTxnSet[len(fundTxnSet)-1].SiacoinOutputID(int(scoFundIndex)) channelTxn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: fundOutputID, UnlockConditions: channelFundingUC, }}, SiacoinOutputs: []types.SiacoinOutput{{ Value: channelSize, UnlockHash: channelAddress, }}, TransactionSignatures: []types.TransactionSignature{{ ParentID: crypto.Hash(fundOutputID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, }}, } // Funding entity creates and signs a transaction that spends the full // channel output. channelOutputID := channelTxn.SiacoinOutputID(0) refundUC, err := cst.wallet.NextAddress() refundAddr := refundUC.UnlockHash() if err != nil { return err } refundTxn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: channelOutputID, UnlockConditions: uc, }}, SiacoinOutputs: []types.SiacoinOutput{{ Value: channelSize, UnlockHash: refundAddr, }}, TransactionSignatures: []types.TransactionSignature{{ ParentID: crypto.Hash(channelOutputID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, }}, } sigHash := refundTxn.SigHash(0) cryptoSig1, err := crypto.SignHash(sigHash, sk1) if err != nil { return err } refundTxn.TransactionSignatures[0].Signature = cryptoSig1[:] // Receiving entity signs the transaction that spends the full channel // output, but with a timelock. refundTxn.TransactionSignatures = append(refundTxn.TransactionSignatures, types.TransactionSignature{ ParentID: crypto.Hash(channelOutputID), PublicKeyIndex: 1, Timelock: cst.cs.dbBlockHeight() + 2, CoveredFields: types.CoveredFields{WholeTransaction: true}, }) sigHash = refundTxn.SigHash(1) cryptoSig2, err := crypto.SignHash(sigHash, sk2) if err != nil { return err } refundTxn.TransactionSignatures[1].Signature = cryptoSig2[:] // Funding entity will now sign and broadcast the funding transaction. sigHash = channelTxn.SigHash(0) cryptoSig0, err := crypto.SignHash(sigHash, channelFundingSK) if err != nil { return err } channelTxn.TransactionSignatures[0].Signature = cryptoSig0[:] err = cst.tpool.AcceptTransactionSet(append(fundTxnSet, channelTxn)) if err != nil { return err } // Put the txn in a block. _, err = cst.miner.AddBlock() if err != nil { return err } // Try to submit the refund transaction before the timelock has expired. err = cst.tpool.AcceptTransactionSet([]types.Transaction{refundTxn}) if err != types.ErrPrematureSignature { return err } // Create a transaction that has partially used the channel, and submit it // to the blockchain to close the channel. closeTxn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: channelOutputID, UnlockConditions: uc, }}, SiacoinOutputs: []types.SiacoinOutput{ { Value: channelSize.Sub(types.NewCurrency64(5)), UnlockHash: refundAddr, }, { Value: types.NewCurrency64(5), }, }, TransactionSignatures: []types.TransactionSignature{ { ParentID: crypto.Hash(channelOutputID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, }, { ParentID: crypto.Hash(channelOutputID), PublicKeyIndex: 1, CoveredFields: types.CoveredFields{WholeTransaction: true}, }, }, } sigHash = closeTxn.SigHash(0) cryptoSig3, err := crypto.SignHash(sigHash, sk1) if err != nil { return err } closeTxn.TransactionSignatures[0].Signature = cryptoSig3[:] sigHash = closeTxn.SigHash(1) cryptoSig4, err := crypto.SignHash(sigHash, sk2) if err != nil { return err } closeTxn.TransactionSignatures[1].Signature = cryptoSig4[:] err = cst.tpool.AcceptTransactionSet([]types.Transaction{closeTxn}) if err != nil { return err } // Mine the block with the transaction. _, err = cst.miner.AddBlock() if err != nil { return err } closeRefundID := closeTxn.SiacoinOutputID(0) closePaymentID := closeTxn.SiacoinOutputID(1) exists := cst.cs.db.inSiacoinOutputs(closeRefundID) if !exists { return errors.New("close txn refund output doesn't exist") } exists = cst.cs.db.inSiacoinOutputs(closePaymentID) if !exists { return errors.New("close txn payment output doesn't exist") } // Create a payment channel where the receiving entity never responds to // the initial transaction. { // Funding entity creates but does not sign a transaction that funds the // channel address. Because the wallet is not very flexible, the channel // txn needs to be fully custom. To get a custom txn, manually create an // address and then use the wallet to fund that address. channelSize := types.NewCurrency64(10e3) channelFundingSK, channelFundingPK, err := crypto.StdKeyGen.Generate() if err != nil { return err } channelFundingUC := types.UnlockConditions{ PublicKeys: []types.SiaPublicKey{{ Algorithm: types.SignatureEd25519, Key: channelFundingPK[:], }}, SignaturesRequired: 1, } channelFundingAddr := channelFundingUC.UnlockHash() fundTxnBuilder := cst.wallet.StartTransaction() err = fundTxnBuilder.FundSiacoins(channelSize) if err != nil { return err } scoFundIndex := fundTxnBuilder.AddSiacoinOutput(types.SiacoinOutput{Value: channelSize, UnlockHash: channelFundingAddr}) fundTxnSet, err := fundTxnBuilder.Sign(true) if err != nil { return err } fundOutputID := fundTxnSet[len(fundTxnSet)-1].SiacoinOutputID(int(scoFundIndex)) channelTxn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: fundOutputID, UnlockConditions: channelFundingUC, }}, SiacoinOutputs: []types.SiacoinOutput{{ Value: channelSize, UnlockHash: channelAddress, }}, TransactionSignatures: []types.TransactionSignature{{ ParentID: crypto.Hash(fundOutputID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, }}, } // Funding entity creates and signs a transaction that spends the full // channel output. channelOutputID := channelTxn.SiacoinOutputID(0) refundUC, err := cst.wallet.NextAddress() refundAddr := refundUC.UnlockHash() if err != nil { return err } refundTxn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: channelOutputID, UnlockConditions: uc, }}, SiacoinOutputs: []types.SiacoinOutput{{ Value: channelSize, UnlockHash: refundAddr, }}, TransactionSignatures: []types.TransactionSignature{{ ParentID: crypto.Hash(channelOutputID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, }}, } sigHash := refundTxn.SigHash(0) cryptoSig1, err := crypto.SignHash(sigHash, sk1) if err != nil { return err } refundTxn.TransactionSignatures[0].Signature = cryptoSig1[:] // Receiving entity never communitcates, funding entity must reclaim // the 'channelSize' coins that were intended to go to the channel. reclaimUC, err := cst.wallet.NextAddress() reclaimAddr := reclaimUC.UnlockHash() if err != nil { return err } reclaimTxn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: fundOutputID, UnlockConditions: channelFundingUC, }}, SiacoinOutputs: []types.SiacoinOutput{{ Value: channelSize, UnlockHash: reclaimAddr, }}, TransactionSignatures: []types.TransactionSignature{{ ParentID: crypto.Hash(fundOutputID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, }}, } sigHash = reclaimTxn.SigHash(0) cryptoSig, err := crypto.SignHash(sigHash, channelFundingSK) if err != nil { return err } reclaimTxn.TransactionSignatures[0].Signature = cryptoSig[:] err = cst.tpool.AcceptTransactionSet(append(fundTxnSet, reclaimTxn)) if err != nil { return err } block, _ := cst.miner.FindBlock() err = cst.cs.AcceptBlock(block) if err != nil { return err } reclaimOutputID := reclaimTxn.SiacoinOutputID(0) exists := cst.cs.db.inSiacoinOutputs(reclaimOutputID) if !exists { return errors.New("failed to reclaim an output that belongs to the funding entity") } } // Create a channel and the open the channel, but close the channel using // the timelocked signature. { // Funding entity creates but does not sign a transaction that funds the // channel address. Because the wallet is not very flexible, the channel // txn needs to be fully custom. To get a custom txn, manually create an // address and then use the wallet to fund that address. channelSize := types.NewCurrency64(10e3) channelFundingSK, channelFundingPK, err := crypto.StdKeyGen.Generate() if err != nil { return err } channelFundingUC := types.UnlockConditions{ PublicKeys: []types.SiaPublicKey{{ Algorithm: types.SignatureEd25519, Key: channelFundingPK[:], }}, SignaturesRequired: 1, } channelFundingAddr := channelFundingUC.UnlockHash() fundTxnBuilder := cst.wallet.StartTransaction() err = fundTxnBuilder.FundSiacoins(channelSize) if err != nil { return err } scoFundIndex := fundTxnBuilder.AddSiacoinOutput(types.SiacoinOutput{Value: channelSize, UnlockHash: channelFundingAddr}) fundTxnSet, err := fundTxnBuilder.Sign(true) if err != nil { return err } fundOutputID := fundTxnSet[len(fundTxnSet)-1].SiacoinOutputID(int(scoFundIndex)) channelTxn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: fundOutputID, UnlockConditions: channelFundingUC, }}, SiacoinOutputs: []types.SiacoinOutput{{ Value: channelSize, UnlockHash: channelAddress, }}, TransactionSignatures: []types.TransactionSignature{{ ParentID: crypto.Hash(fundOutputID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, }}, } // Funding entity creates and signs a transaction that spends the full // channel output. channelOutputID := channelTxn.SiacoinOutputID(0) refundUC, err := cst.wallet.NextAddress() refundAddr := refundUC.UnlockHash() if err != nil { return err } refundTxn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: channelOutputID, UnlockConditions: uc, }}, SiacoinOutputs: []types.SiacoinOutput{{ Value: channelSize, UnlockHash: refundAddr, }}, TransactionSignatures: []types.TransactionSignature{{ ParentID: crypto.Hash(channelOutputID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, }}, } sigHash := refundTxn.SigHash(0) cryptoSig1, err := crypto.SignHash(sigHash, sk1) if err != nil { return err } refundTxn.TransactionSignatures[0].Signature = cryptoSig1[:] // Receiving entity signs the transaction that spends the full channel // output, but with a timelock. refundTxn.TransactionSignatures = append(refundTxn.TransactionSignatures, types.TransactionSignature{ ParentID: crypto.Hash(channelOutputID), PublicKeyIndex: 1, Timelock: cst.cs.dbBlockHeight() + 2, CoveredFields: types.CoveredFields{WholeTransaction: true}, }) sigHash = refundTxn.SigHash(1) cryptoSig2, err := crypto.SignHash(sigHash, sk2) if err != nil { return err } refundTxn.TransactionSignatures[1].Signature = cryptoSig2[:] // Funding entity will now sign and broadcast the funding transaction. sigHash = channelTxn.SigHash(0) cryptoSig0, err := crypto.SignHash(sigHash, channelFundingSK) if err != nil { return err } channelTxn.TransactionSignatures[0].Signature = cryptoSig0[:] err = cst.tpool.AcceptTransactionSet(append(fundTxnSet, channelTxn)) if err != nil { return err } // Put the txn in a block. block, _ := cst.miner.FindBlock() err = cst.cs.AcceptBlock(block) if err != nil { return err } // Receiving entity never signs another transaction, so the funding // entity waits until the timelock is complete, and then submits the // refundTxn. for i := 0; i < 3; i++ { block, _ := cst.miner.FindBlock() err = cst.cs.AcceptBlock(block) if err != nil { return err } } err = cst.tpool.AcceptTransactionSet([]types.Transaction{refundTxn}) if err != nil { return err } block, _ = cst.miner.FindBlock() err = cst.cs.AcceptBlock(block) if err != nil { return err } refundOutputID := refundTxn.SiacoinOutputID(0) exists := cst.cs.db.inSiacoinOutputs(refundOutputID) if !exists { return errors.New("timelocked refund transaction did not get spent correctly") } } return nil } */ /* // TestPaymentChannelBlocks creates a consensus set tester and uses it to call // testPaymentChannelBlocks. func TestPaymentChannelBlocks(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestPaymentChannelBlocks") if err != nil { t.Fatal(err) } defer cst.closeCst() err = cst.testPaymentChannelBlocks() if err != nil { t.Fatal(err) } } */ Sia-1.0.4/modules/consensus/applytransaction.go000066400000000000000000000167241302033070500216450ustar00rootroot00000000000000package consensus // applytransaction.go handles applying a transaction to the consensus set. // There is an assumption that the transaction has already been verified. import ( "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // applySiacoinInputs takes all of the siacoin inputs in a transaction and // applies them to the state, updating the diffs in the processed block. func applySiacoinInputs(tx *bolt.Tx, pb *processedBlock, t types.Transaction) { // Remove all siacoin inputs from the unspent siacoin outputs list. for _, sci := range t.SiacoinInputs { sco, err := getSiacoinOutput(tx, sci.ParentID) if build.DEBUG && err != nil { panic(err) } scod := modules.SiacoinOutputDiff{ Direction: modules.DiffRevert, ID: sci.ParentID, SiacoinOutput: sco, } pb.SiacoinOutputDiffs = append(pb.SiacoinOutputDiffs, scod) commitSiacoinOutputDiff(tx, scod, modules.DiffApply) } } // applySiacoinOutputs takes all of the siacoin outputs in a transaction and // applies them to the state, updating the diffs in the processed block. func applySiacoinOutputs(tx *bolt.Tx, pb *processedBlock, t types.Transaction) { // Add all siacoin outputs to the unspent siacoin outputs list. for i, sco := range t.SiacoinOutputs { scoid := t.SiacoinOutputID(uint64(i)) scod := modules.SiacoinOutputDiff{ Direction: modules.DiffApply, ID: scoid, SiacoinOutput: sco, } pb.SiacoinOutputDiffs = append(pb.SiacoinOutputDiffs, scod) commitSiacoinOutputDiff(tx, scod, modules.DiffApply) } } // applyFileContracts iterates through all of the file contracts in a // transaction and applies them to the state, updating the diffs in the proccesed // block. func applyFileContracts(tx *bolt.Tx, pb *processedBlock, t types.Transaction) { for i, fc := range t.FileContracts { fcid := t.FileContractID(uint64(i)) fcd := modules.FileContractDiff{ Direction: modules.DiffApply, ID: fcid, FileContract: fc, } pb.FileContractDiffs = append(pb.FileContractDiffs, fcd) commitFileContractDiff(tx, fcd, modules.DiffApply) // Get the portion of the contract that goes into the siafund pool and // add it to the siafund pool. sfp := getSiafundPool(tx) sfpd := modules.SiafundPoolDiff{ Direction: modules.DiffApply, Previous: sfp, Adjusted: sfp.Add(types.Tax(blockHeight(tx), fc.Payout)), } pb.SiafundPoolDiffs = append(pb.SiafundPoolDiffs, sfpd) commitSiafundPoolDiff(tx, sfpd, modules.DiffApply) } } // applyTxFileContractRevisions iterates through all of the file contract // revisions in a transaction and applies them to the state, updating the diffs // in the processed block. func applyFileContractRevisions(tx *bolt.Tx, pb *processedBlock, t types.Transaction) { for _, fcr := range t.FileContractRevisions { fc, err := getFileContract(tx, fcr.ParentID) if build.DEBUG && err != nil { panic(err) } // Add the diff to delete the old file contract. fcd := modules.FileContractDiff{ Direction: modules.DiffRevert, ID: fcr.ParentID, FileContract: fc, } pb.FileContractDiffs = append(pb.FileContractDiffs, fcd) commitFileContractDiff(tx, fcd, modules.DiffApply) // Add the diff to add the revised file contract. newFC := types.FileContract{ FileSize: fcr.NewFileSize, FileMerkleRoot: fcr.NewFileMerkleRoot, WindowStart: fcr.NewWindowStart, WindowEnd: fcr.NewWindowEnd, Payout: fc.Payout, ValidProofOutputs: fcr.NewValidProofOutputs, MissedProofOutputs: fcr.NewMissedProofOutputs, UnlockHash: fcr.NewUnlockHash, RevisionNumber: fcr.NewRevisionNumber, } fcd = modules.FileContractDiff{ Direction: modules.DiffApply, ID: fcr.ParentID, FileContract: newFC, } pb.FileContractDiffs = append(pb.FileContractDiffs, fcd) commitFileContractDiff(tx, fcd, modules.DiffApply) } } // applyTxStorageProofs iterates through all of the storage proofs in a // transaction and applies them to the state, updating the diffs in the processed // block. func applyStorageProofs(tx *bolt.Tx, pb *processedBlock, t types.Transaction) { for _, sp := range t.StorageProofs { fc, err := getFileContract(tx, sp.ParentID) if build.DEBUG && err != nil { panic(err) } // Add all of the outputs in the ValidProofOutputs of the contract. for i, vpo := range fc.ValidProofOutputs { spoid := sp.ParentID.StorageProofOutputID(types.ProofValid, uint64(i)) dscod := modules.DelayedSiacoinOutputDiff{ Direction: modules.DiffApply, ID: spoid, SiacoinOutput: vpo, MaturityHeight: pb.Height + types.MaturityDelay, } pb.DelayedSiacoinOutputDiffs = append(pb.DelayedSiacoinOutputDiffs, dscod) commitDelayedSiacoinOutputDiff(tx, dscod, modules.DiffApply) } fcd := modules.FileContractDiff{ Direction: modules.DiffRevert, ID: sp.ParentID, FileContract: fc, } pb.FileContractDiffs = append(pb.FileContractDiffs, fcd) commitFileContractDiff(tx, fcd, modules.DiffApply) } } // applyTxSiafundInputs takes all of the siafund inputs in a transaction and // applies them to the state, updating the diffs in the processed block. func applySiafundInputs(tx *bolt.Tx, pb *processedBlock, t types.Transaction) { for _, sfi := range t.SiafundInputs { // Calculate the volume of siacoins to put in the claim output. sfo, err := getSiafundOutput(tx, sfi.ParentID) if build.DEBUG && err != nil { panic(err) } claimPortion := getSiafundPool(tx).Sub(sfo.ClaimStart).Div(types.SiafundCount).Mul(sfo.Value) // Add the claim output to the delayed set of outputs. sco := types.SiacoinOutput{ Value: claimPortion, UnlockHash: sfi.ClaimUnlockHash, } sfoid := sfi.ParentID.SiaClaimOutputID() dscod := modules.DelayedSiacoinOutputDiff{ Direction: modules.DiffApply, ID: sfoid, SiacoinOutput: sco, MaturityHeight: pb.Height + types.MaturityDelay, } pb.DelayedSiacoinOutputDiffs = append(pb.DelayedSiacoinOutputDiffs, dscod) commitDelayedSiacoinOutputDiff(tx, dscod, modules.DiffApply) // Create the siafund output diff and remove the output from the // consensus set. sfod := modules.SiafundOutputDiff{ Direction: modules.DiffRevert, ID: sfi.ParentID, SiafundOutput: sfo, } pb.SiafundOutputDiffs = append(pb.SiafundOutputDiffs, sfod) commitSiafundOutputDiff(tx, sfod, modules.DiffApply) } } // applySiafundOutput applies a siafund output to the consensus set. func applySiafundOutputs(tx *bolt.Tx, pb *processedBlock, t types.Transaction) { for i, sfo := range t.SiafundOutputs { sfoid := t.SiafundOutputID(uint64(i)) sfo.ClaimStart = getSiafundPool(tx) sfod := modules.SiafundOutputDiff{ Direction: modules.DiffApply, ID: sfoid, SiafundOutput: sfo, } pb.SiafundOutputDiffs = append(pb.SiafundOutputDiffs, sfod) commitSiafundOutputDiff(tx, sfod, modules.DiffApply) } } // applyTransaction applies the contents of a transaction to the ConsensusSet. // This produces a set of diffs, which are stored in the blockNode containing // the transaction. No verification is done by this function. func applyTransaction(tx *bolt.Tx, pb *processedBlock, t types.Transaction) { applySiacoinInputs(tx, pb, t) applySiacoinOutputs(tx, pb, t) applyFileContracts(tx, pb, t) applyFileContractRevisions(tx, pb, t) applyStorageProofs(tx, pb, t) applySiafundInputs(tx, pb, t) applySiafundOutputs(tx, pb, t) } Sia-1.0.4/modules/consensus/applytransaction_test.go000066400000000000000000000567041302033070500227060ustar00rootroot00000000000000package consensus /* import ( "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // TestApplySiacoinInputs probes the applySiacoinInputs method of the consensus // set. func TestApplySiacoinInputs(t *testing.T) { if testing.Short() { t.SkipNow() } // Create a consensus set and get it to 3 siacoin outputs. The consensus // set starts with 2 siacoin outputs, mining a block will add another. cst, err := createConsensusSetTester("TestApplySiacoinInputs") if err != nil { t.Fatal(err) } defer cst.closeCst() b, _ := cst.miner.FindBlock() err = cst.cs.AcceptBlock(b) if err != nil { t.Fatal(err) } // Create a block node to use with application. pb := new(processedBlock) // Fetch the output id's of each siacoin output in the consensus set. var ids []types.SiacoinOutputID cst.cs.db.forEachSiacoinOutputs(func(id types.SiacoinOutputID, sco types.SiacoinOutput) { ids = append(ids, id) }) // Apply a transaction with a single siacoin input. txn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{ {ParentID: ids[0]}, }, } cst.cs.applySiacoinInputs(pb, txn) exists := cst.cs.db.inSiacoinOutputs(ids[0]) if exists { t.Error("Failed to conusme a siacoin output") } if cst.cs.db.lenSiacoinOutputs() != 2 { t.Error("siacoin outputs not correctly updated") } if len(pb.SiacoinOutputDiffs) != 1 { t.Error("block node was not updated for single transaction") } if pb.SiacoinOutputDiffs[0].Direction != modules.DiffRevert { t.Error("wrong diff direction applied when consuming a siacoin output") } if pb.SiacoinOutputDiffs[0].ID != ids[0] { t.Error("wrong id used when consuming a siacoin output") } // Apply a transaction with two siacoin inputs. txn = types.Transaction{ SiacoinInputs: []types.SiacoinInput{ {ParentID: ids[1]}, {ParentID: ids[2]}, }, } cst.cs.applySiacoinInputs(pb, txn) if cst.cs.db.lenSiacoinOutputs() != 0 { t.Error("failed to consume all siacoin outputs in the consensus set") } if len(pb.SiacoinOutputDiffs) != 3 { t.Error("processed block was not updated for single transaction") } } // TestMisuseApplySiacoinInputs misuses applySiacoinInput and checks that a // panic was triggered. func TestMisuseApplySiacoinInputs(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestMisuseApplySiacoinInputs") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node to use with application. pb := new(processedBlock) // Fetch the output id's of each siacoin output in the consensus set. var ids []types.SiacoinOutputID cst.cs.db.forEachSiacoinOutputs(func(id types.SiacoinOutputID, sco types.SiacoinOutput) { ids = append(ids, id) }) // Apply a transaction with a single siacoin input. txn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{ {ParentID: ids[0]}, }, } cst.cs.applySiacoinInputs(pb, txn) // Trigger the panic that occurs when an output is applied incorrectly, and // perform a catch to read the error that is created. defer func() { r := recover() if r == nil { t.Error("expecting error after corrupting database") } }() cst.cs.applySiacoinInputs(pb, txn) } // TestApplySiacoinOutputs probes the applySiacoinOutput method of the // consensus set. func TestApplySiacoinOutputs(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestApplySiacoinOutputs") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node to use with application. pb := new(processedBlock) // Apply a transaction with a single siacoin output. txn := types.Transaction{ SiacoinOutputs: []types.SiacoinOutput{{}}, } cst.cs.applySiacoinOutputs(pb, txn) scoid := txn.SiacoinOutputID(0) exists := cst.cs.db.inSiacoinOutputs(scoid) if !exists { t.Error("Failed to create siacoin output") } if cst.cs.db.lenSiacoinOutputs() != 3 { // 3 because createConsensusSetTester has 2 initially. t.Error("siacoin outputs not correctly updated") } if len(pb.SiacoinOutputDiffs) != 1 { t.Error("block node was not updated for single element transaction") } if pb.SiacoinOutputDiffs[0].Direction != modules.DiffApply { t.Error("wrong diff direction applied when creating a siacoin output") } if pb.SiacoinOutputDiffs[0].ID != scoid { t.Error("wrong id used when creating a siacoin output") } // Apply a transaction with 2 siacoin outputs. txn = types.Transaction{ SiacoinOutputs: []types.SiacoinOutput{ {Value: types.NewCurrency64(1)}, {Value: types.NewCurrency64(2)}, }, } cst.cs.applySiacoinOutputs(pb, txn) scoid0 := txn.SiacoinOutputID(0) scoid1 := txn.SiacoinOutputID(1) exists = cst.cs.db.inSiacoinOutputs(scoid0) if !exists { t.Error("Failed to create siacoin output") } exists = cst.cs.db.inSiacoinOutputs(scoid1) if !exists { t.Error("Failed to create siacoin output") } if cst.cs.db.lenSiacoinOutputs() != 5 { // 5 because createConsensusSetTester has 2 initially. t.Error("siacoin outputs not correctly updated") } if len(pb.SiacoinOutputDiffs) != 3 { t.Error("block node was not updated correctly") } } // TestMisuseApplySiacoinOutputs misuses applySiacoinOutputs and checks that a // panic was triggered. func TestMisuseApplySiacoinOutputs(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestMisuseApplySiacoinOutputs") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node to use with application. pb := new(processedBlock) // Apply a transaction with a single siacoin output. txn := types.Transaction{ SiacoinOutputs: []types.SiacoinOutput{{}}, } cst.cs.applySiacoinOutputs(pb, txn) // Trigger the panic that occurs when an output is applied incorrectly, and // perform a catch to read the error that is created. defer func() { r := recover() if r == nil { t.Error("no panic occurred when misusing applySiacoinInput") } }() cst.cs.applySiacoinOutputs(pb, txn) } // TestApplyFileContracts probes the applyFileContracts method of the // consensus set. func TestApplyFileContracts(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestApplyFileContracts") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node to use with application. pb := new(processedBlock) // Apply a transaction with a single file contract. txn := types.Transaction{ FileContracts: []types.FileContract{{}}, } cst.cs.applyFileContracts(pb, txn) fcid := txn.FileContractID(0) exists := cst.cs.db.inFileContracts(fcid) if !exists { t.Error("Failed to create file contract") } if cst.cs.db.lenFileContracts() != 1 { t.Error("file contracts not correctly updated") } if len(pb.FileContractDiffs) != 1 { t.Error("block node was not updated for single element transaction") } if pb.FileContractDiffs[0].Direction != modules.DiffApply { t.Error("wrong diff direction applied when creating a file contract") } if pb.FileContractDiffs[0].ID != fcid { t.Error("wrong id used when creating a file contract") } // Apply a transaction with 2 file contracts. txn = types.Transaction{ FileContracts: []types.FileContract{ {Payout: types.NewCurrency64(1)}, {Payout: types.NewCurrency64(300e3)}, }, } cst.cs.applyFileContracts(pb, txn) fcid0 := txn.FileContractID(0) fcid1 := txn.FileContractID(1) exists = cst.cs.db.inFileContracts(fcid0) if !exists { t.Error("Failed to create file contract") } exists = cst.cs.db.inFileContracts(fcid1) if !exists { t.Error("Failed to create file contract") } if cst.cs.db.lenFileContracts() != 3 { t.Error("file contracts not correctly updated") } if len(pb.FileContractDiffs) != 3 { t.Error("block node was not updated correctly") } if cst.cs.siafundPool.Cmp64(10e3) != 0 { t.Error("siafund pool did not update correctly upon creation of a file contract") } } // TestMisuseApplyFileContracts misuses applyFileContracts and checks that a // panic was triggered. func TestMisuseApplyFileContracts(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestMisuseApplyFileContracts") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node to use with application. pb := new(processedBlock) // Apply a transaction with a single file contract. txn := types.Transaction{ FileContracts: []types.FileContract{{}}, } cst.cs.applyFileContracts(pb, txn) // Trigger the panic that occurs when an output is applied incorrectly, and // perform a catch to read the error that is created. defer func() { r := recover() if r == nil { t.Error("no panic occurred when misusing applySiacoinInput") } }() cst.cs.applyFileContracts(pb, txn) } // TestApplyFileContractRevisions probes the applyFileContractRevisions method // of the consensus set. func TestApplyFileContractRevisions(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestApplyFileContractRevisions") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node to use with application. pb := new(processedBlock) // Apply a transaction with two file contracts - that way there is // something to revise. txn := types.Transaction{ FileContracts: []types.FileContract{ {}, {Payout: types.NewCurrency64(1)}, }, } cst.cs.applyFileContracts(pb, txn) fcid0 := txn.FileContractID(0) fcid1 := txn.FileContractID(1) // Apply a single file contract revision. txn = types.Transaction{ FileContractRevisions: []types.FileContractRevision{ { ParentID: fcid0, NewFileSize: 1, }, }, } cst.cs.applyFileContractRevisions(pb, txn) exists := cst.cs.db.inFileContracts(fcid0) if !exists { t.Error("Revision killed a file contract") } fc := cst.cs.db.getFileContracts(fcid0) if fc.FileSize != 1 { t.Error("file contract filesize not properly updated") } if cst.cs.db.lenFileContracts() != 2 { t.Error("file contracts not correctly updated") } if len(pb.FileContractDiffs) != 4 { // 2 creating the initial contracts, 1 to remove the old, 1 to add the revision. t.Error("block node was not updated for single element transaction") } if pb.FileContractDiffs[2].Direction != modules.DiffRevert { t.Error("wrong diff direction applied when revising a file contract") } if pb.FileContractDiffs[3].Direction != modules.DiffApply { t.Error("wrong diff direction applied when revising a file contract") } if pb.FileContractDiffs[2].ID != fcid0 { t.Error("wrong id used when revising a file contract") } if pb.FileContractDiffs[3].ID != fcid0 { t.Error("wrong id used when revising a file contract") } // Apply a transaction with 2 file contract revisions. txn = types.Transaction{ FileContractRevisions: []types.FileContractRevision{ { ParentID: fcid0, NewFileSize: 2, }, { ParentID: fcid1, NewFileSize: 3, }, }, } cst.cs.applyFileContractRevisions(pb, txn) exists = cst.cs.db.inFileContracts(fcid0) if !exists { t.Error("Revision ate file contract") } fc0 := cst.cs.db.getFileContracts(fcid0) exists = cst.cs.db.inFileContracts(fcid1) if !exists { t.Error("Revision ate file contract") } fc1 := cst.cs.db.getFileContracts(fcid1) if fc0.FileSize != 2 { t.Error("Revision not correctly applied") } if fc1.FileSize != 3 { t.Error("Revision not correctly applied") } if cst.cs.db.lenFileContracts() != 2 { t.Error("file contracts not correctly updated") } if len(pb.FileContractDiffs) != 8 { t.Error("block node was not updated correctly") } } // TestMisuseApplyFileContractRevisions misuses applyFileContractRevisions and // checks that a panic was triggered. func TestMisuseApplyFileContractRevisions(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestMisuseApplyFileContractRevisions") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node to use with application. pb := new(processedBlock) // Trigger a panic from revising a nonexistent file contract. defer func() { r := recover() if r != errNilItem { t.Error("no panic occurred when misusing applySiacoinInput") } }() txn := types.Transaction{ FileContractRevisions: []types.FileContractRevision{{}}, } cst.cs.applyFileContractRevisions(pb, txn) } // TestApplyStorageProofs probes the applyStorageProofs method of the consensus // set. func TestApplyStorageProofs(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestApplyStorageProofs") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node to use with application. pb := new(processedBlock) pb.Height = cst.cs.height() // Apply a transaction with two file contracts - there is a reason to // create a storage proof. txn := types.Transaction{ FileContracts: []types.FileContract{ { Payout: types.NewCurrency64(300e3), ValidProofOutputs: []types.SiacoinOutput{ {Value: types.NewCurrency64(290e3)}, }, }, {}, { Payout: types.NewCurrency64(600e3), ValidProofOutputs: []types.SiacoinOutput{ {Value: types.NewCurrency64(280e3)}, {Value: types.NewCurrency64(300e3)}, }, }, }, } cst.cs.applyFileContracts(pb, txn) fcid0 := txn.FileContractID(0) fcid1 := txn.FileContractID(1) fcid2 := txn.FileContractID(2) // Apply a single storage proof. txn = types.Transaction{ StorageProofs: []types.StorageProof{{ParentID: fcid0}}, } cst.cs.applyStorageProofs(pb, txn) exists := cst.cs.db.inFileContracts(fcid0) if exists { t.Error("Storage proof did not disable a file contract.") } if cst.cs.db.lenFileContracts() != 2 { t.Error("file contracts not correctly updated") } if len(pb.FileContractDiffs) != 4 { // 3 creating the initial contracts, 1 for the storage proof. t.Error("block node was not updated for single element transaction") } if pb.FileContractDiffs[3].Direction != modules.DiffRevert { t.Error("wrong diff direction applied when revising a file contract") } if pb.FileContractDiffs[3].ID != fcid0 { t.Error("wrong id used when revising a file contract") } spoid0 := fcid0.StorageProofOutputID(types.ProofValid, 0) exists = cst.cs.db.inDelayedSiacoinOutputsHeight(pb.Height+types.MaturityDelay, spoid0) if !exists { t.Error("storage proof output not created after applying a storage proof") } sco := cst.cs.db.getDelayedSiacoinOutputs(pb.Height+types.MaturityDelay, spoid0) if sco.Value.Cmp64(290e3) != 0 { t.Error("storage proof output was created with the wrong value") } // Apply a transaction with 2 storage proofs. txn = types.Transaction{ StorageProofs: []types.StorageProof{ {ParentID: fcid1}, {ParentID: fcid2}, }, } cst.cs.applyStorageProofs(pb, txn) exists = cst.cs.db.inFileContracts(fcid1) if exists { t.Error("Storage proof failed to consume file contract.") } exists = cst.cs.db.inFileContracts(fcid2) if exists { t.Error("storage proof did not consume file contract") } if cst.cs.db.lenFileContracts() != 0 { t.Error("file contracts not correctly updated") } if len(pb.FileContractDiffs) != 6 { t.Error("block node was not updated correctly") } spoid1 := fcid1.StorageProofOutputID(types.ProofValid, 0) exists = cst.cs.db.inSiacoinOutputs(spoid1) if exists { t.Error("output created when file contract had no corresponding output") } spoid2 := fcid2.StorageProofOutputID(types.ProofValid, 0) exists = cst.cs.db.inDelayedSiacoinOutputsHeight(pb.Height+types.MaturityDelay, spoid2) if !exists { t.Error("no output created by first output of file contract") } sco = cst.cs.db.getDelayedSiacoinOutputs(pb.Height+types.MaturityDelay, spoid2) if sco.Value.Cmp64(280e3) != 0 { t.Error("first siacoin output created has wrong value") } spoid3 := fcid2.StorageProofOutputID(types.ProofValid, 1) exists = cst.cs.db.inDelayedSiacoinOutputsHeight(pb.Height+types.MaturityDelay, spoid3) if !exists { t.Error("second output not created for storage proof") } sco = cst.cs.db.getDelayedSiacoinOutputs(pb.Height+types.MaturityDelay, spoid3) if sco.Value.Cmp64(300e3) != 0 { t.Error("second siacoin output has wrong value") } if cst.cs.siafundPool.Cmp64(30e3) != 0 { t.Error("siafund pool not being added up correctly") } } // TestNonexistentStorageProof applies a storage proof which points to a // nonextentent parent. func TestNonexistentStorageProof(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestNonexistentStorageProof") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node to use with application. pb := new(processedBlock) // Trigger a panic by applying a storage proof for a nonexistent file // contract. defer func() { r := recover() if r != errNilItem { t.Error("no panic occurred when misusing applySiacoinInput") } }() txn := types.Transaction{ StorageProofs: []types.StorageProof{{}}, } cst.cs.applyStorageProofs(pb, txn) } // TestDuplicateStorageProof applies a storage proof which has already been // applied. func TestDuplicateStorageProof(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestDuplicateStorageProof") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node. pb := new(processedBlock) pb.Height = cst.cs.height() // Create a file contract for the storage proof to prove. txn0 := types.Transaction{ FileContracts: []types.FileContract{ { Payout: types.NewCurrency64(300e3), ValidProofOutputs: []types.SiacoinOutput{ {Value: types.NewCurrency64(290e3)}, }, }, }, } cst.cs.applyFileContracts(pb, txn0) fcid := txn0.FileContractID(0) // Apply a single storage proof. txn1 := types.Transaction{ StorageProofs: []types.StorageProof{{ParentID: fcid}}, } cst.cs.applyStorageProofs(pb, txn1) // Trigger a panic by applying the storage proof again. defer func() { r := recover() if r != ErrDuplicateValidProofOutput { t.Error("failed to trigger ErrDuplicateValidProofOutput:", r) } }() cst.cs.applyFileContracts(pb, txn0) // File contract was consumed by the first proof. cst.cs.applyStorageProofs(pb, txn1) } // TestApplySiafundInputs probes the applySiafundInputs method of the consensus // set. func TestApplySiafundInputs(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestApplySiafundInputs") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node to use with application. pb := new(processedBlock) pb.Height = cst.cs.height() // Fetch the output id's of each siacoin output in the consensus set. var ids []types.SiafundOutputID cst.cs.db.forEachSiafundOutputs(func(sfoid types.SiafundOutputID, sfo types.SiafundOutput) { ids = append(ids, sfoid) }) // Apply a transaction with a single siafund input. txn := types.Transaction{ SiafundInputs: []types.SiafundInput{ {ParentID: ids[0]}, }, } cst.cs.applySiafundInputs(pb, txn) exists := cst.cs.db.inSiafundOutputs(ids[0]) if exists { t.Error("Failed to conusme a siafund output") } if cst.cs.db.lenSiafundOutputs() != 2 { t.Error("siafund outputs not correctly updated", cst.cs.db.lenSiafundOutputs()) } if len(pb.SiafundOutputDiffs) != 1 { t.Error("block node was not updated for single transaction") } if pb.SiafundOutputDiffs[0].Direction != modules.DiffRevert { t.Error("wrong diff direction applied when consuming a siafund output") } if pb.SiafundOutputDiffs[0].ID != ids[0] { t.Error("wrong id used when consuming a siafund output") } if cst.cs.db.lenDelayedSiacoinOutputsHeight(cst.cs.height()+types.MaturityDelay) != 2 { // 1 for a block subsidy, 1 for the siafund claim. t.Error("siafund claim was not created") } } // TestMisuseApplySiafundInputs misuses applySiafundInputs and checks that a // panic was triggered. func TestMisuseApplySiafundInputs(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestApplySiacoinInput") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node to use with application. pb := new(processedBlock) pb.Height = cst.cs.height() // Fetch the output id's of each siacoin output in the consensus set. var ids []types.SiafundOutputID cst.cs.db.forEachSiafundOutputs(func(sfoid types.SiafundOutputID, sfo types.SiafundOutput) { ids = append(ids, sfoid) }) // Apply a transaction with a single siafund input. txn := types.Transaction{ SiafundInputs: []types.SiafundInput{ {ParentID: ids[0]}, }, } cst.cs.applySiafundInputs(pb, txn) // Trigger the panic that occurs when an output is applied incorrectly, and // perform a catch to read the error that is created. defer func() { r := recover() if r != ErrMisuseApplySiafundInput { t.Error("no panic occurred when misusing applySiacoinInput") t.Error(r) } }() cst.cs.applySiafundInputs(pb, txn) } // TestApplySiafundOutputs probes the applySiafundOutputs method of the // consensus set. func TestApplySiafundOutputs(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestApplySiafundOutputs") if err != nil { t.Fatal(err) } defer cst.closeCst() cst.cs.siafundPool = types.NewCurrency64(101) // Create a block node to use with application. pb := new(processedBlock) // Apply a transaction with a single siafund output. txn := types.Transaction{ SiafundOutputs: []types.SiafundOutput{{}}, } cst.cs.applySiafundOutputs(pb, txn) sfoid := txn.SiafundOutputID(0) exists := cst.cs.db.inSiafundOutputs(sfoid) if !exists { t.Error("Failed to create siafund output") } if cst.cs.db.lenSiafundOutputs() != 4 { t.Error("siafund outputs not correctly updated") } if len(pb.SiafundOutputDiffs) != 1 { t.Error("block node was not updated for single element transaction") } if pb.SiafundOutputDiffs[0].Direction != modules.DiffApply { t.Error("wrong diff direction applied when creating a siafund output") } if pb.SiafundOutputDiffs[0].ID != sfoid { t.Error("wrong id used when creating a siafund output") } if pb.SiafundOutputDiffs[0].SiafundOutput.ClaimStart.Cmp64(101) != 0 { t.Error("claim start set incorrectly when creating a siafund output") } // Apply a transaction with 2 siacoin outputs. txn = types.Transaction{ SiafundOutputs: []types.SiafundOutput{ {Value: types.NewCurrency64(1)}, {Value: types.NewCurrency64(2)}, }, } cst.cs.applySiafundOutputs(pb, txn) sfoid0 := txn.SiafundOutputID(0) sfoid1 := txn.SiafundOutputID(1) exists = cst.cs.db.inSiafundOutputs(sfoid0) if !exists { t.Error("Failed to create siafund output") } exists = cst.cs.db.inSiafundOutputs(sfoid1) if !exists { t.Error("Failed to create siafund output") } if cst.cs.db.lenSiafundOutputs() != 6 { t.Error("siafund outputs not correctly updated") } if len(pb.SiafundOutputDiffs) != 3 { t.Error("block node was not updated for single element transaction") } } // TestMisuseApplySiafundOutputs misuses applySiafundOutputs and checks that a // panic was triggered. func TestMisuseApplySiafundOutputs(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestMisuseApplySiafundOutputs") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node to use with application. pb := new(processedBlock) // Apply a transaction with a single siacoin output. txn := types.Transaction{ SiafundOutputs: []types.SiafundOutput{{}}, } cst.cs.applySiafundOutputs(pb, txn) // Trigger the panic that occurs when an output is applied incorrectly, and // perform a catch to read the error that is created. defer func() { r := recover() if r != ErrMisuseApplySiafundOutput { t.Error("no panic occurred when misusing applySiafundInput") } }() cst.cs.applySiafundOutputs(pb, txn) } */ Sia-1.0.4/modules/consensus/block_rules.go000066400000000000000000000035411302033070500205470ustar00rootroot00000000000000package consensus import ( "sort" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/types" ) // blockRuleHelper assists with block validity checks by calculating values // on blocks that are relevant to validity rules. type blockRuleHelper interface { minimumValidChildTimestamp(dbBucket, *processedBlock) types.Timestamp } // stdBlockRuleHelper is the standard implementation of blockRuleHelper. type stdBlockRuleHelper struct{} // minimumValidChildTimestamp returns the earliest timestamp that a child node // can have while still being valid. See section 'Block Timestamps' in // Consensus.md. // // To boost performance, minimumValidChildTimestamp is passed a bucket that it // can use from inside of a boltdb transaction. func (rh stdBlockRuleHelper) minimumValidChildTimestamp(blockMap dbBucket, pb *processedBlock) types.Timestamp { // Get the previous MedianTimestampWindow timestamps. windowTimes := make(types.TimestampSlice, types.MedianTimestampWindow) windowTimes[0] = pb.Block.Timestamp parent := pb.Block.ParentID for i := uint64(1); i < types.MedianTimestampWindow; i++ { // If the genesis block is 'parent', use the genesis block timestamp // for all remaining times. if parent == (types.BlockID{}) { windowTimes[i] = windowTimes[i-1] continue } // Get the next parent's bytes. Because the ordering is specific, the // parent does not need to be decoded entirely to get the desired // information. This provides a performance boost. The id of the next // parent lies at the first 32 bytes, and the timestamp of the block // lies at bytes 40-48. parentBytes := blockMap.Get(parent[:]) copy(parent[:], parentBytes[:32]) windowTimes[i] = types.Timestamp(encoding.DecUint64(parentBytes[40:48])) } sort.Sort(windowTimes) // Return the median of the sorted timestamps. return windowTimes[len(windowTimes)/2] } Sia-1.0.4/modules/consensus/block_validation.go000066400000000000000000000071031302033070500215450ustar00rootroot00000000000000package consensus import ( "bytes" "errors" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( errBadMinerPayouts = errors.New("miner payout sum does not equal block subsidy") errEarlyTimestamp = errors.New("block timestamp is too early") errExtremeFutureTimestamp = errors.New("block timestamp too far in future, discarded") errFutureTimestamp = errors.New("block timestamp too far in future, but saved for later use") errLargeBlock = errors.New("block is too large to be accepted") ) // blockValidator validates a Block against a set of block validity rules. type blockValidator interface { // ValidateBlock validates a block against a minimum timestamp, a block // target, and a block height. ValidateBlock(types.Block, types.Timestamp, types.Target, types.BlockHeight) error } // stdBlockValidator is the standard implementation of blockValidator. type stdBlockValidator struct { // clock is a Clock interface that indicates the current system time. clock types.Clock // marshaler encodes and decodes between objects and byte slices. marshaler encoding.GenericMarshaler } // NewBlockValidator creates a new stdBlockValidator with default settings. func NewBlockValidator() stdBlockValidator { return stdBlockValidator{ clock: types.StdClock{}, marshaler: encoding.StdGenericMarshaler{}, } } // checkMinerPayouts compares a block's miner payouts to the block's subsidy and // returns true if they are equal. func checkMinerPayouts(b types.Block, height types.BlockHeight) bool { // Add up the payouts and check that all values are legal. var payoutSum types.Currency for _, payout := range b.MinerPayouts { if payout.Value.IsZero() { return false } payoutSum = payoutSum.Add(payout.Value) } return b.CalculateSubsidy(height).Equals(payoutSum) } // checkTarget returns true if the block's ID meets the given target. func checkTarget(b types.Block, target types.Target) bool { blockHash := b.ID() return bytes.Compare(target[:], blockHash[:]) >= 0 } // ValidateBlock validates a block against a minimum timestamp, a block target, // and a block height. Returns nil if the block is valid and an appropriate // error otherwise. func (bv stdBlockValidator) ValidateBlock(b types.Block, minTimestamp types.Timestamp, target types.Target, height types.BlockHeight) error { // Check that the timestamp is not too far in the past to be acceptable. if minTimestamp > b.Timestamp { return errEarlyTimestamp } // Check that the target of the new block is sufficient. if !checkTarget(b, target) { return modules.ErrBlockUnsolved } // Check that the block is below the size limit. if uint64(len(bv.marshaler.Marshal(b))) > types.BlockSizeLimit { return errLargeBlock } // Check if the block is in the extreme future. We make a distinction between // future and extreme future because there is an assumption that by the time // the extreme future arrives, this block will no longer be a part of the // longest fork because it will have been ignored by all of the miners. if b.Timestamp > bv.clock.Now()+types.ExtremeFutureThreshold { return errExtremeFutureTimestamp } // Verify that the miner payouts are valid. if !checkMinerPayouts(b, height) { return errBadMinerPayouts } // Check if the block is in the near future, but too far to be acceptable. // This is the last check because it's an expensive check, and not worth // performing if the payouts are incorrect. if b.Timestamp > bv.clock.Now()+types.FutureThreshold { return errFutureTimestamp } return nil } Sia-1.0.4/modules/consensus/block_validation_test.go000066400000000000000000000105171302033070500226070ustar00rootroot00000000000000package consensus import ( "testing" "github.com/NebulousLabs/Sia/types" ) // mockMarshaler is a mock implementation of the encoding.GenericMarshaler // interface that allows the client to pre-define the length of the marshaled // data. type mockMarshaler struct { marshalLength uint64 } // Marshal marshals an object into an empty byte slice of marshalLength. func (m mockMarshaler) Marshal(interface{}) []byte { return make([]byte, m.marshalLength) } // Unmarshal is not implemented. func (m mockMarshaler) Unmarshal([]byte, interface{}) error { panic("not implemented") } // mockClock is a mock implementation of the types.Clock interface that allows // the client to pre-define a return value for Now(). type mockClock struct { now types.Timestamp } // Now returns mockClock's pre-defined Timestamp. func (c mockClock) Now() types.Timestamp { return c.now } var validateBlockTests = []struct { now types.Timestamp minTimestamp types.Timestamp blockTimestamp types.Timestamp blockSize uint64 errWant error msg string }{ { minTimestamp: types.Timestamp(5), blockTimestamp: types.Timestamp(4), errWant: errEarlyTimestamp, msg: "ValidateBlock should reject blocks with timestamps that are too early", }, { blockSize: types.BlockSizeLimit + 1, errWant: errLargeBlock, msg: "ValidateBlock should reject excessively large blocks", }, { now: types.Timestamp(50), blockTimestamp: types.Timestamp(50) + types.ExtremeFutureThreshold + 1, errWant: errExtremeFutureTimestamp, msg: "ValidateBlock should reject blocks timestamped in the extreme future", }, } // TestUnitValidateBlock runs a series of unit tests for ValidateBlock. func TestUnitValidateBlock(t *testing.T) { // TODO(mtlynch): Populate all parameters to ValidateBlock so that everything // is valid except for the attribute that causes validation to fail. (i.e. // don't assume an ordering to the implementation of the validation function). for _, tt := range validateBlockTests { b := types.Block{ Timestamp: tt.blockTimestamp, } blockValidator := stdBlockValidator{ marshaler: mockMarshaler{ marshalLength: tt.blockSize, }, clock: mockClock{ now: tt.now, }, } err := blockValidator.ValidateBlock(b, tt.minTimestamp, types.RootDepth, 0) if err != tt.errWant { t.Errorf("%s: got %v, want %v", tt.msg, err, tt.errWant) } } } // TestCheckMinerPayouts probes the checkMinerPayouts function. func TestCheckMinerPayouts(t *testing.T) { // All tests are done at height = 0. coinbase := types.CalculateCoinbase(0) // Create a block with a single valid payout. b := types.Block{ MinerPayouts: []types.SiacoinOutput{ {Value: coinbase}, }, } if !checkMinerPayouts(b, 0) { t.Error("payouts evaluated incorrectly when there is only one payout.") } // Try a block with an incorrect payout. b = types.Block{ MinerPayouts: []types.SiacoinOutput{ {Value: coinbase.Sub(types.NewCurrency64(1))}, }, } if checkMinerPayouts(b, 0) { t.Error("payouts evaluated incorrectly when there is a too-small payout") } // Try a block with 2 payouts. b = types.Block{ MinerPayouts: []types.SiacoinOutput{ {Value: coinbase.Sub(types.NewCurrency64(1))}, {Value: types.NewCurrency64(1)}, }, } if !checkMinerPayouts(b, 0) { t.Error("payouts evaluated incorrectly when there are 2 payouts") } // Try a block with 2 payouts that are too large. b = types.Block{ MinerPayouts: []types.SiacoinOutput{ {Value: coinbase}, {Value: coinbase}, }, } if checkMinerPayouts(b, 0) { t.Error("payouts evaluated incorrectly when there are two large payouts") } // Create a block with an empty payout. b = types.Block{ MinerPayouts: []types.SiacoinOutput{ {Value: coinbase}, {}, }, } if checkMinerPayouts(b, 0) { t.Error("payouts evaluated incorrectly when there is only one payout.") } } // TestCheckTarget probes the checkTarget function. func TestCheckTarget(t *testing.T) { var b types.Block lowTarget := types.RootDepth highTarget := types.Target{} sameTarget := types.Target(b.ID()) if !checkTarget(b, lowTarget) { t.Error("CheckTarget failed for a low target") } if checkTarget(b, highTarget) { t.Error("CheckTarget passed for a high target") } if !checkTarget(b, sameTarget) { t.Error("CheckTarget failed for a same target") } } Sia-1.0.4/modules/consensus/changelog.go000066400000000000000000000112441302033070500201710ustar00rootroot00000000000000package consensus // changelog.go implements a persistent changelog in the consenus database // tracking all of the atomic changes to the consensus set. The primary use of // the changelog is for subscribers that have persistence - instead of // subscribing from the very beginning and receiving all changes from genesis // each time the daemon starts up, the subscribers can start from the most // recent change that they are familiar with. // // The changelog is set up as a singley linked list where each change points // forward to the next change. In bolt, the key is a hash of the changeEntry // and the value is a struct containing the changeEntry and the key of the next // changeEntry. The empty hash key leads to the 'changeTail', which contains // the id of the most recent changeEntry. // // Initialization only needs to worry about creating the blank change entry, // the genesis block will call 'append' later on during initialization. import ( "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) var ( // ChangeLog contains a list of atomic changes that have happened to the // consensus set so that subscribers can subscribe from the most recent // change they have seen. ChangeLog = []byte("ChangeLog") // ChangeLogTailID is a key that points to the id of the current changelog // tail. ChangeLogTailID = []byte("ChangeLogTailID") ) type ( // changeEntry records a single atomic change to the consensus set. changeEntry struct { RevertedBlocks []types.BlockID AppliedBlocks []types.BlockID } // changeNode contains a change entry and a pointer to the next change // entry, and is the object that gets stored in the database. changeNode struct { Entry changeEntry Next modules.ConsensusChangeID } ) // appendChangeLog adds a new change entry to the change log. func appendChangeLog(tx *bolt.Tx, ce changeEntry) error { // Insert the change entry. cl := tx.Bucket(ChangeLog) ceid := ce.ID() cn := changeNode{Entry: ce, Next: modules.ConsensusChangeID{}} err := cl.Put(ceid[:], encoding.Marshal(cn)) if err != nil { return err } // Update the tail node to point to the new change entry as the next entry. var tailID modules.ConsensusChangeID copy(tailID[:], cl.Get(ChangeLogTailID)) if tailID != (modules.ConsensusChangeID{}) { // Get the old tail node. var tailCN changeNode tailCNBytes := cl.Get(tailID[:]) err = encoding.Unmarshal(tailCNBytes, &tailCN) if err != nil { return err } // Point the 'next' of the old tail node to the new tail node and // insert. tailCN.Next = ceid err = cl.Put(tailID[:], encoding.Marshal(tailCN)) if err != nil { return err } } // Update the tail id. err = cl.Put(ChangeLogTailID, ceid[:]) if err != nil { return err } return nil } // getEntry returns the change entry with a given id, using a bool to indicate // existence. func getEntry(tx *bolt.Tx, id modules.ConsensusChangeID) (ce changeEntry, exists bool) { var cn changeNode cl := tx.Bucket(ChangeLog) changeNodeBytes := cl.Get(id[:]) if changeNodeBytes == nil { return changeEntry{}, false } err := encoding.Unmarshal(changeNodeBytes, &cn) if build.DEBUG && err != nil { panic(err) } return cn.Entry, true } // ID returns the id of a change entry. func (ce *changeEntry) ID() modules.ConsensusChangeID { return modules.ConsensusChangeID(crypto.HashObject(ce)) } // NextEntry returns the entry after the current entry. func (ce *changeEntry) NextEntry(tx *bolt.Tx) (nextEntry changeEntry, exists bool) { // Get the change node associated with the provided change entry. ceid := ce.ID() var cn changeNode cl := tx.Bucket(ChangeLog) changeNodeBytes := cl.Get(ceid[:]) err := encoding.Unmarshal(changeNodeBytes, &cn) if build.DEBUG && err != nil { panic(err) } return getEntry(tx, cn.Next) } // createChangeLog assumes that no change log exists and creates a new one. func (cs *ConsensusSet) createChangeLog(tx *bolt.Tx) error { // Create the changelog bucket. cl, err := tx.CreateBucket(ChangeLog) if err != nil { return err } // Add the genesis block as the first entry of the change log. ge := cs.genesisEntry() geid := ge.ID() cn := changeNode{ Entry: ge, Next: modules.ConsensusChangeID{}, } err = cl.Put(geid[:], encoding.Marshal(cn)) if err != nil { return err } err = cl.Put(ChangeLogTailID, geid[:]) if err != nil { return err } return nil } // genesisEntry returns the id of the genesis block log entry. func (cs *ConsensusSet) genesisEntry() changeEntry { return changeEntry{ AppliedBlocks: []types.BlockID{cs.blockRoot.Block.ID()}, } } Sia-1.0.4/modules/consensus/changelog_test.go000066400000000000000000000040721302033070500212310ustar00rootroot00000000000000package consensus import ( "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // TestIntegrationChangeLog does a general test of the changelog by creating a // subscriber that subscribes partway into startup and checking that the // correct ordering of blocks are provided. func TestIntegrationChangeLog(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // Get a blank consensus set tester so that the mocked subscriber can join // immediately after genesis. cst, err := blankConsensusSetTester("TestIntegrationChangeLog") if err != nil { t.Fatal(err) } defer cst.Close() // Add a mocked subscriber and check that it receives the correct number of // blocks. ms := newMockSubscriber() cst.cs.ConsensusSetSubscribe(&ms, modules.ConsensusChangeBeginning) if ms.updates[0].AppliedBlocks[0].ID() != cst.cs.blockRoot.Block.ID() { t.Fatal("subscription did not correctly receive the genesis block") } if len(ms.updates) != 1 { t.Fatal("subscription resulted in the wrong number of blocks being sent") } // Create a copy of the subscriber that will subscribe to the consensus at // the tail of the updates. tailSubscriber := ms.copySub() cst.cs.ConsensusSetSubscribe(&tailSubscriber, tailSubscriber.updates[len(tailSubscriber.updates)-1].ID) if len(tailSubscriber.updates) != 1 { t.Fatal("subscription resulted in the wrong number of blocks being sent") } // Create a copy of the subscriber that will join when it is not at 0, but it is behind. behindSubscriber := ms.copySub() cst.addSiafunds() cst.mineSiacoins() cst.cs.ConsensusSetSubscribe(&behindSubscriber, behindSubscriber.updates[len(behindSubscriber.updates)-1].ID) if types.BlockHeight(len(behindSubscriber.updates)) != cst.cs.dbBlockHeight()+1 { t.Fatal("subscription resulted in the wrong number of blocks being sent") } if len(ms.updates) != len(tailSubscriber.updates) { t.Error("subscribers have inconsistent update chains") } if len(ms.updates) != len(behindSubscriber.updates) { t.Error("subscribers have inconsistent update chains") } } Sia-1.0.4/modules/consensus/consensusdb.go000066400000000000000000000402001302033070500205620ustar00rootroot00000000000000package consensus // consensusdb.go contains all of the functions related to performing consensus // related actions on the database, including initializing the consensus // portions of the database. Many errors cause panics instead of being handled // gracefully, but only when the debug flag is set. The errors are silently // ignored otherwise, which is suboptimal. import ( "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) var ( prefixDSCO = []byte("dsco_") prefixFCEX = []byte("fcex_") // BlockHeight is a bucket that stores the current block height. // // Generally we would just look at BlockPath.Stats(), but there is an error // in boltdb that prevents the bucket stats from updating until a tx is // committed. Wasn't a problem until we started doing the entire block as // one tx. // // DEPRECATED - block.Stats() should be sufficient to determine the block // height, but currently stats are only computed after committing a // transaction, therefore cannot be assumed reliable. BlockHeight = []byte("BlockHeight") // BlockMap is a database bucket containing all of the processed blocks, // keyed by their id. This includes blocks that are not currently in the // consensus set, and blocks that may not have been fully validated yet. BlockMap = []byte("BlockMap") // BlockPath is a database bucket containing a mapping from the height of a // block to the id of the block at that height. BlockPath only includes // blocks in the current path. BlockPath = []byte("BlockPath") // Consistency is a database bucket with a flag indicating whether // inconsistencies within the database have been detected. Consistency = []byte("Consistency") // SiacoinOutputs is a database bucket that contains all of the unspent // siacoin outputs. SiacoinOutputs = []byte("SiacoinOutputs") // FileContracts is a database bucket that contains all of the open file // contracts. FileContracts = []byte("FileContracts") // SiafundOutputs is a database bucket that contains all of the unspent // siafund outputs. SiafundOutputs = []byte("SiafundOutputs") // SiafundPool is a database bucket storing the current value of the // siafund pool. SiafundPool = []byte("SiafundPool") ) // createConsensusObjects initialzes the consensus portions of the database. func (cs *ConsensusSet) createConsensusDB(tx *bolt.Tx) error { // Enumerate and create the database buckets. buckets := [][]byte{ BlockHeight, BlockMap, BlockPath, Consistency, SiacoinOutputs, FileContracts, SiafundOutputs, SiafundPool, } for _, bucket := range buckets { _, err := tx.CreateBucket(bucket) if err != nil { return err } } // Set the block height to -1, so the genesis block is at height 0. blockHeight := tx.Bucket(BlockHeight) underflow := types.BlockHeight(0) err := blockHeight.Put(BlockHeight, encoding.Marshal(underflow-1)) if err != nil { return err } // Set the siafund pool to 0. setSiafundPool(tx, types.NewCurrency64(0)) // Update the siafund output diffs map for the genesis block on disk. This // needs to happen between the database being opened/initilized and the // consensus set hash being calculated for _, sfod := range cs.blockRoot.SiafundOutputDiffs { commitSiafundOutputDiff(tx, sfod, modules.DiffApply) } // Add the miner payout from the genesis block to the delayed siacoin // outputs - unspendable, as the unlock hash is blank. createDSCOBucket(tx, types.MaturityDelay) addDSCO(tx, types.MaturityDelay, cs.blockRoot.Block.MinerPayoutID(0), types.SiacoinOutput{ Value: types.CalculateCoinbase(0), UnlockHash: types.UnlockHash{}, }) // Add the genesis block to the block strucutres - checksum must be taken // after pushing the genesis block into the path. pushPath(tx, cs.blockRoot.Block.ID()) if build.DEBUG { cs.blockRoot.ConsensusChecksum = consensusChecksum(tx) } addBlockMap(tx, &cs.blockRoot) return nil } // blockHeight returns the height of the blockchain. func blockHeight(tx *bolt.Tx) types.BlockHeight { var height types.BlockHeight bh := tx.Bucket(BlockHeight) err := encoding.Unmarshal(bh.Get(BlockHeight), &height) if build.DEBUG && err != nil { panic(err) } return height } // currentBlockID returns the id of the most recent block in the consensus set. func currentBlockID(tx *bolt.Tx) types.BlockID { id, err := getPath(tx, blockHeight(tx)) if build.DEBUG && err != nil { panic(err) } return id } // currentProcessedBlock returns the most recent block in the consensus set. func currentProcessedBlock(tx *bolt.Tx) *processedBlock { pb, err := getBlockMap(tx, currentBlockID(tx)) if build.DEBUG && err != nil { panic(err) } return pb } // getBlockMap returns a processed block with the input id. func getBlockMap(tx *bolt.Tx, id types.BlockID) (*processedBlock, error) { // Look up the encoded block. pbBytes := tx.Bucket(BlockMap).Get(id[:]) if pbBytes == nil { return nil, errNilItem } // Decode the block - should never fail. var pb processedBlock err := encoding.Unmarshal(pbBytes, &pb) if build.DEBUG && err != nil { panic(err) } return &pb, nil } // addBlockMap adds a processed block to the block map. func addBlockMap(tx *bolt.Tx, pb *processedBlock) { id := pb.Block.ID() err := tx.Bucket(BlockMap).Put(id[:], encoding.Marshal(*pb)) if build.DEBUG && err != nil { panic(err) } } // getPath returns the block id at 'height' in the block path. func getPath(tx *bolt.Tx, height types.BlockHeight) (id types.BlockID, err error) { idBytes := tx.Bucket(BlockPath).Get(encoding.Marshal(height)) if idBytes == nil { return types.BlockID{}, errNilItem } err = encoding.Unmarshal(idBytes, &id) if build.DEBUG && err != nil { panic(err) } return id, nil } // pushPath adds a block to the BlockPath at current height + 1. func pushPath(tx *bolt.Tx, bid types.BlockID) { // Fetch and update the block height. bh := tx.Bucket(BlockHeight) heightBytes := bh.Get(BlockHeight) var oldHeight types.BlockHeight err := encoding.Unmarshal(heightBytes, &oldHeight) if build.DEBUG && err != nil { panic(err) } newHeightBytes := encoding.Marshal(oldHeight + 1) err = bh.Put(BlockHeight, newHeightBytes) if build.DEBUG && err != nil { panic(err) } // Add the block to the block path. bp := tx.Bucket(BlockPath) err = bp.Put(newHeightBytes, bid[:]) if build.DEBUG && err != nil { panic(err) } } // popPath removes a block from the "end" of the chain, i.e. the block // with the largest height. func popPath(tx *bolt.Tx) { // Fetch and update the block height. bh := tx.Bucket(BlockHeight) oldHeightBytes := bh.Get(BlockHeight) var oldHeight types.BlockHeight err := encoding.Unmarshal(oldHeightBytes, &oldHeight) if build.DEBUG && err != nil { panic(err) } newHeightBytes := encoding.Marshal(oldHeight - 1) err = bh.Put(BlockHeight, newHeightBytes) if build.DEBUG && err != nil { panic(err) } // Remove the block from the path - make sure to remove the block at // oldHeight. bp := tx.Bucket(BlockPath) err = bp.Delete(oldHeightBytes) if build.DEBUG && err != nil { panic(err) } } // isSiacoinOutput returns true if there is a siacoin output of that id in the // database. func isSiacoinOutput(tx *bolt.Tx, id types.SiacoinOutputID) bool { bucket := tx.Bucket(SiacoinOutputs) sco := bucket.Get(id[:]) return sco != nil } // getSiacoinOutput fetches a siacoin output from the database. An error is // returned if the siacoin output does not exist. func getSiacoinOutput(tx *bolt.Tx, id types.SiacoinOutputID) (types.SiacoinOutput, error) { scoBytes := tx.Bucket(SiacoinOutputs).Get(id[:]) if scoBytes == nil { return types.SiacoinOutput{}, errNilItem } var sco types.SiacoinOutput err := encoding.Unmarshal(scoBytes, &sco) if err != nil { return types.SiacoinOutput{}, err } return sco, nil } // addSiacoinOutput adds a siacoin output to the database. An error is returned // if the siacoin output is already in the database. func addSiacoinOutput(tx *bolt.Tx, id types.SiacoinOutputID, sco types.SiacoinOutput) { // While this is not supposed to be allowed, there's a bug in the consensus // code which means that earlier versions have accetped 0-value outputs // onto the blockchain. A hardfork to remove 0-value outputs will fix this, // and that hardfork is planned, but not yet. /* if build.DEBUG && sco.Value.IsZero() { panic("discovered a zero value siacoin output") } */ siacoinOutputs := tx.Bucket(SiacoinOutputs) // Sanity check - should not be adding an item that exists. if build.DEBUG && siacoinOutputs.Get(id[:]) != nil { panic("repeat siacoin output") } err := siacoinOutputs.Put(id[:], encoding.Marshal(sco)) if build.DEBUG && err != nil { panic(err) } } // removeSiacoinOutput removes a siacoin output from the database. An error is // returned if the siacoin output is not in the database prior to removal. func removeSiacoinOutput(tx *bolt.Tx, id types.SiacoinOutputID) { scoBucket := tx.Bucket(SiacoinOutputs) // Sanity check - should not be removing an item that is not in the db. if build.DEBUG && scoBucket.Get(id[:]) == nil { panic("nil siacoin output") } err := scoBucket.Delete(id[:]) if build.DEBUG && err != nil { panic(err) } } // getFileContract fetches a file contract from the database, returning an // error if it is not there. func getFileContract(tx *bolt.Tx, id types.FileContractID) (fc types.FileContract, err error) { fcBytes := tx.Bucket(FileContracts).Get(id[:]) if fcBytes == nil { return types.FileContract{}, errNilItem } err = encoding.Unmarshal(fcBytes, &fc) if err != nil { return types.FileContract{}, err } return fc, nil } // addFileContract adds a file contract to the database. An error is returned // if the file contract is already in the database. func addFileContract(tx *bolt.Tx, id types.FileContractID, fc types.FileContract) { // Add the file contract to the database. fcBucket := tx.Bucket(FileContracts) // Sanity check - should not be adding a zero-payout file contract. if build.DEBUG && fc.Payout.IsZero() { panic("adding zero-payout file contract") } // Sanity check - should not be adding a file contract already in the db. if build.DEBUG && fcBucket.Get(id[:]) != nil { panic("repeat file contract") } err := fcBucket.Put(id[:], encoding.Marshal(fc)) if build.DEBUG && err != nil { panic(err) } // Add an entry for when the file contract expires. expirationBucketID := append(prefixFCEX, encoding.Marshal(fc.WindowEnd)...) expirationBucket, err := tx.CreateBucketIfNotExists(expirationBucketID) if build.DEBUG && err != nil { panic(err) } err = expirationBucket.Put(id[:], []byte{}) if build.DEBUG && err != nil { panic(err) } } // removeFileContract removes a file contract from the database. func removeFileContract(tx *bolt.Tx, id types.FileContractID) { // Delete the file contract entry. fcBucket := tx.Bucket(FileContracts) fcBytes := fcBucket.Get(id[:]) // Sanity check - should not be removing a file contract not in the db. if build.DEBUG && fcBytes == nil { panic("nil file contract") } err := fcBucket.Delete(id[:]) if build.DEBUG && err != nil { panic(err) } // Delete the entry for the file contract's expiration. The portion of // 'fcBytes' used to determine the expiration bucket id is the // byte-representation of the file contract window end, which always // appears at bytes 48-56. expirationBucketID := append(prefixFCEX, fcBytes[48:56]...) expirationBucket := tx.Bucket(expirationBucketID) expirationBytes := expirationBucket.Get(id[:]) if expirationBytes == nil { panic(errNilItem) } err = expirationBucket.Delete(id[:]) if build.DEBUG && err != nil { panic(err) } } // getSiafundOutput fetches a siafund output from the database. An error is // returned if the siafund output does not exist. func getSiafundOutput(tx *bolt.Tx, id types.SiafundOutputID) (types.SiafundOutput, error) { sfoBytes := tx.Bucket(SiafundOutputs).Get(id[:]) if sfoBytes == nil { return types.SiafundOutput{}, errNilItem } var sfo types.SiafundOutput err := encoding.Unmarshal(sfoBytes, &sfo) if err != nil { return types.SiafundOutput{}, err } return sfo, nil } // addSiafundOutput adds a siafund output to the database. An error is returned // if the siafund output is already in the database. func addSiafundOutput(tx *bolt.Tx, id types.SiafundOutputID, sfo types.SiafundOutput) { siafundOutputs := tx.Bucket(SiafundOutputs) // Sanity check - should not be adding a siafund output with a value of // zero. if build.DEBUG && sfo.Value.IsZero() { panic("zero value siafund being added") } // Sanity check - should not be adding an item already in the db. if build.DEBUG && siafundOutputs.Get(id[:]) != nil { panic("repeat siafund output") } err := siafundOutputs.Put(id[:], encoding.Marshal(sfo)) if build.DEBUG && err != nil { panic(err) } } // removeSiafundOutput removes a siafund output from the database. An error is // returned if the siafund output is not in the database prior to removal. func removeSiafundOutput(tx *bolt.Tx, id types.SiafundOutputID) { sfoBucket := tx.Bucket(SiafundOutputs) if build.DEBUG && sfoBucket.Get(id[:]) == nil { panic("nil siafund output") } err := sfoBucket.Delete(id[:]) if build.DEBUG && err != nil { panic(err) } } // getSiafundPool returns the current value of the siafund pool. No error is // returned as the siafund pool should always be available. func getSiafundPool(tx *bolt.Tx) (pool types.Currency) { bucket := tx.Bucket(SiafundPool) poolBytes := bucket.Get(SiafundPool) // An error should only be returned if the object stored in the siafund // pool bucket is either unavailable or otherwise malformed. As this is a // developer error, a panic is appropriate. err := encoding.Unmarshal(poolBytes, &pool) if build.DEBUG && err != nil { panic(err) } return pool } // setSiafundPool updates the saved siafund pool on disk func setSiafundPool(tx *bolt.Tx, c types.Currency) { err := tx.Bucket(SiafundPool).Put(SiafundPool, encoding.Marshal(c)) if build.DEBUG && err != nil { panic(err) } } // addDSCO adds a delayed siacoin output to the consnesus set. func addDSCO(tx *bolt.Tx, bh types.BlockHeight, id types.SiacoinOutputID, sco types.SiacoinOutput) { // Sanity check - dsco should never have a value of zero. // An error in the consensus code means sometimes there are 0-value dscos // in the blockchain. A hardfork will fix this. /* if build.DEBUG && sco.Value.IsZero() { panic("zero-value dsco being added") } */ // Sanity check - output should not already be in the full set of outputs. if build.DEBUG && tx.Bucket(SiacoinOutputs).Get(id[:]) != nil { panic("dsco already in output set") } dscoBucketID := append(prefixDSCO, encoding.EncUint64(uint64(bh))...) dscoBucket := tx.Bucket(dscoBucketID) // Sanity check - should not be adding an item already in the db. if build.DEBUG && dscoBucket.Get(id[:]) != nil { panic(errRepeatInsert) } err := dscoBucket.Put(id[:], encoding.Marshal(sco)) if build.DEBUG && err != nil { panic(err) } } // removeDSCO removes a delayed siacoin output from the consensus set. func removeDSCO(tx *bolt.Tx, bh types.BlockHeight, id types.SiacoinOutputID) { bucketID := append(prefixDSCO, encoding.Marshal(bh)...) // Sanity check - should not remove an item not in the db. dscoBucket := tx.Bucket(bucketID) if build.DEBUG && dscoBucket.Get(id[:]) == nil { panic("nil dsco") } err := dscoBucket.Delete(id[:]) if build.DEBUG && err != nil { panic(err) } } // createDSCOBucket creates a bucket for the delayed siacoin outputs at the // input height. func createDSCOBucket(tx *bolt.Tx, bh types.BlockHeight) { bucketID := append(prefixDSCO, encoding.Marshal(bh)...) _, err := tx.CreateBucket(bucketID) if build.DEBUG && err != nil { panic(err) } } // deleteDSCOBucket deletes the bucket that held a set of delayed siacoin // outputs. func deleteDSCOBucket(tx *bolt.Tx, bh types.BlockHeight) { // Delete the bucket. bucketID := append(prefixDSCO, encoding.Marshal(bh)...) bucket := tx.Bucket(bucketID) if build.DEBUG && bucket == nil { panic(errNilBucket) } // TODO: Check that the bucket is empty. Using Stats() does not work at the // moment, as there is an error in the boltdb code. err := tx.DeleteBucket(bucketID) if build.DEBUG && err != nil { panic(err) } } Sia-1.0.4/modules/consensus/consensusdb_helpers_test.go000066400000000000000000000157321302033070500233570ustar00rootroot00000000000000package consensus // database_test.go contains a bunch of legacy functions to preserve // compatibility with the test suite. import ( "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // dbBlockHeight is a convenience function allowing blockHeight to be called // without a bolt.Tx. func (cs *ConsensusSet) dbBlockHeight() (bh types.BlockHeight) { dbErr := cs.db.View(func(tx *bolt.Tx) error { bh = blockHeight(tx) return nil }) if dbErr != nil { panic(dbErr) } return bh } // dbCurrentBlockID is a convenience function allowing currentBlockID to be // called without a bolt.Tx. func (cs *ConsensusSet) dbCurrentBlockID() (id types.BlockID) { dbErr := cs.db.View(func(tx *bolt.Tx) error { id = currentBlockID(tx) return nil }) if dbErr != nil { panic(dbErr) } return id } // dbCurrentProcessedBlock is a convenience function allowing // currentProcessedBlock to be called without a bolt.Tx. func (cs *ConsensusSet) dbCurrentProcessedBlock() (pb *processedBlock) { dbErr := cs.db.View(func(tx *bolt.Tx) error { pb = currentProcessedBlock(tx) return nil }) if dbErr != nil { panic(dbErr) } return pb } // dbGetPath is a convenience function allowing getPath to be called without a // bolt.Tx. func (cs *ConsensusSet) dbGetPath(bh types.BlockHeight) (id types.BlockID, err error) { dbErr := cs.db.View(func(tx *bolt.Tx) error { id, err = getPath(tx, bh) return nil }) if dbErr != nil { panic(dbErr) } return id, err } // dbPushPath is a convenience function allowing pushPath to be called without a // bolt.Tx. func (cs *ConsensusSet) dbPushPath(bid types.BlockID) { dbErr := cs.db.Update(func(tx *bolt.Tx) error { pushPath(tx, bid) return nil }) if dbErr != nil { panic(dbErr) } } // dbGetBlockMap is a convenience function allowing getBlockMap to be called // without a bolt.Tx. func (cs *ConsensusSet) dbGetBlockMap(id types.BlockID) (pb *processedBlock, err error) { dbErr := cs.db.View(func(tx *bolt.Tx) error { pb, err = getBlockMap(tx, id) return nil }) if dbErr != nil { panic(dbErr) } return pb, err } // dbGetSiacoinOutput is a convenience function allowing getSiacoinOutput to be // called without a bolt.Tx. func (cs *ConsensusSet) dbGetSiacoinOutput(id types.SiacoinOutputID) (sco types.SiacoinOutput, err error) { dbErr := cs.db.View(func(tx *bolt.Tx) error { sco, err = getSiacoinOutput(tx, id) return nil }) if dbErr != nil { panic(dbErr) } return sco, err } // getArbSiacoinOutput is a convenience function fetching a single random // siacoin output from the database. func (cs *ConsensusSet) getArbSiacoinOutput() (scoid types.SiacoinOutputID, sco types.SiacoinOutput, err error) { dbErr := cs.db.View(func(tx *bolt.Tx) error { cursor := tx.Bucket(SiacoinOutputs).Cursor() scoidBytes, scoBytes := cursor.First() copy(scoid[:], scoidBytes) return encoding.Unmarshal(scoBytes, &sco) }) if dbErr != nil { panic(dbErr) } if err != nil { return types.SiacoinOutputID{}, types.SiacoinOutput{}, err } return scoid, sco, nil } // dbGetFileContract is a convenience function allowing getFileContract to be // called without a bolt.Tx. func (cs *ConsensusSet) dbGetFileContract(id types.FileContractID) (fc types.FileContract, err error) { dbErr := cs.db.View(func(tx *bolt.Tx) error { fc, err = getFileContract(tx, id) return nil }) if dbErr != nil { panic(dbErr) } return fc, err } // dbAddFileContract is a convenience function allowing addFileContract to be // called without a bolt.Tx. func (cs *ConsensusSet) dbAddFileContract(id types.FileContractID, fc types.FileContract) { dbErr := cs.db.Update(func(tx *bolt.Tx) error { addFileContract(tx, id, fc) return nil }) if dbErr != nil { panic(dbErr) } } // dbRemoveFileContract is a convenience function allowing removeFileContract // to be called without a bolt.Tx. func (cs *ConsensusSet) dbRemoveFileContract(id types.FileContractID) { dbErr := cs.db.Update(func(tx *bolt.Tx) error { removeFileContract(tx, id) return nil }) if dbErr != nil { panic(dbErr) } } // dbGetSiafundOutput is a convenience function allowing getSiafundOutput to be // called without a bolt.Tx. func (cs *ConsensusSet) dbGetSiafundOutput(id types.SiafundOutputID) (sfo types.SiafundOutput, err error) { dbErr := cs.db.View(func(tx *bolt.Tx) error { sfo, err = getSiafundOutput(tx, id) return nil }) if dbErr != nil { panic(dbErr) } return sfo, err } // dbAddSiafundOutput is a convenience function allowing addSiafundOutput to be // called without a bolt.Tx. func (cs *ConsensusSet) dbAddSiafundOutput(id types.SiafundOutputID, sfo types.SiafundOutput) { dbErr := cs.db.Update(func(tx *bolt.Tx) error { addSiafundOutput(tx, id, sfo) return nil }) if dbErr != nil { panic(dbErr) } } // dbGetSiafundPool is a convenience function allowing getSiafundPool to be // called without a bolt.Tx. func (cs *ConsensusSet) dbGetSiafundPool() (siafundPool types.Currency) { dbErr := cs.db.View(func(tx *bolt.Tx) error { siafundPool = getSiafundPool(tx) return nil }) if dbErr != nil { panic(dbErr) } return siafundPool } // dbGetDSCO is a convenience function allowing a delayed siacoin output to be // fetched without a bolt.Tx. An error is returned if the delayed output is not // found at the maturity height indicated by the input. func (cs *ConsensusSet) dbGetDSCO(height types.BlockHeight, id types.SiacoinOutputID) (dsco types.SiacoinOutput, err error) { dbErr := cs.db.View(func(tx *bolt.Tx) error { dscoBucketID := append(prefixDSCO, encoding.Marshal(height)...) dscoBucket := tx.Bucket(dscoBucketID) if dscoBucket == nil { err = errNilItem return nil } dscoBytes := dscoBucket.Get(id[:]) if dscoBytes == nil { err = errNilItem return nil } err = encoding.Unmarshal(dscoBytes, &dsco) if err != nil { panic(err) } return nil }) if dbErr != nil { panic(dbErr) } return dsco, err } // dbStorageProofSegment is a convenience function allowing // 'storageProofSegment' to be called during testing without a tx. func (cs *ConsensusSet) dbStorageProofSegment(fcid types.FileContractID) (index uint64, err error) { dbErr := cs.db.View(func(tx *bolt.Tx) error { index, err = storageProofSegment(tx, fcid) return nil }) if dbErr != nil { panic(dbErr) } return index, err } // dbValidStorageProofs is a convenience function allowing 'validStorageProofs' // to be called during testing without a tx. func (cs *ConsensusSet) dbValidStorageProofs(t types.Transaction) (err error) { dbErr := cs.db.View(func(tx *bolt.Tx) error { err = validStorageProofs(tx, t) return nil }) if dbErr != nil { panic(dbErr) } return err } // dbValidFileContractRevisions is a convenience function allowing // 'validFileContractRevisions' to be called during testing without a tx. func (cs *ConsensusSet) dbValidFileContractRevisions(t types.Transaction) (err error) { dbErr := cs.db.View(func(tx *bolt.Tx) error { err = validFileContractRevisions(tx, t) return nil }) if dbErr != nil { panic(dbErr) } return err } Sia-1.0.4/modules/consensus/consensusset.go000066400000000000000000000235541302033070500210050ustar00rootroot00000000000000package consensus // All changes to the consenuss set are made via diffs, specifically by calling // a commitDiff function. This means that future modifications (such as // replacing in-memory versions of the utxo set with on-disk versions of the // utxo set) should be relatively easy to verify for correctness. Modifying the // commitDiff functions will be sufficient. import ( "errors" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/sync" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" "github.com/NebulousLabs/demotemutex" ) var ( errNilGateway = errors.New("cannot have a nil gateway as input") ) // The ConsensusSet is the object responsible for tracking the current status // of the blockchain. Broadly speaking, it is responsible for maintaining // consensus. It accepts blocks and constructs a blockchain, forking when // necessary. type ConsensusSet struct { // The gateway manages peer connections and keeps the consensus set // synchronized to the rest of the network. gateway modules.Gateway // The block root contains the genesis block. blockRoot processedBlock // Subscribers to the consensus set will receive a changelog every time // there is an update to the consensus set. At initialization, they receive // all changes that they are missing. // // Memory: A consensus set typically has fewer than 10 subscribers, and // subscription typically happens entirely at startup. This slice is // unlikely to grow beyond 1kb, and cannot by manipulated by an attacker as // the function of adding a subscriber should not be exposed. subscribers []modules.ConsensusSetSubscriber // dosBlocks are blocks that are invalid, but the invalidity is only // discoverable during an expensive step of validation. These blocks are // recorded to eliminate a DoS vector where an expensive-to-validate block // is submitted to the consensus set repeatedly. // // TODO: dosBlocks needs to be moved into the database, and if there's some // reason it can't be in THE database, it should be in a separate database. // dosBlocks is an unbounded map that an attacker can manipulate, though // iirc manipulations are expensive, to the tune of creating a blockchain // PoW per DoS block (though the attacker could conceivably build off of // the genesis block, meaning the PoW is not very expensive. dosBlocks map[types.BlockID]struct{} // checkingConsistency is a bool indicating whether or not a consistency // check is in progress. The consistency check logic call itself, resulting // in infinite loops. This bool prevents that while still allowing for full // granularity consistency checks. Previously, consistency checks were only // performed after a full reorg, but now they are performed after every // block. checkingConsistency bool // synced is true if initial blockchain download has finished. It indicates // whether the consensus set is synced with the network. synced bool // Interfaces to abstract the dependencies of the ConsensusSet. marshaler encoding.GenericMarshaler blockRuleHelper blockRuleHelper blockValidator blockValidator // Utilities db *persist.BoltDatabase log *persist.Logger mu demotemutex.DemoteMutex persistDir string tg sync.ThreadGroup } // New returns a new ConsensusSet, containing at least the genesis block. If // there is an existing block database present in the persist directory, it // will be loaded. func New(gateway modules.Gateway, bootstrap bool, persistDir string) (*ConsensusSet, error) { // Check for nil dependencies. if gateway == nil { return nil, errNilGateway } // Create the ConsensusSet object. cs := &ConsensusSet{ gateway: gateway, blockRoot: processedBlock{ Block: types.GenesisBlock, ChildTarget: types.RootTarget, Depth: types.RootDepth, DiffsGenerated: true, }, dosBlocks: make(map[types.BlockID]struct{}), marshaler: encoding.StdGenericMarshaler{}, blockRuleHelper: stdBlockRuleHelper{}, blockValidator: NewBlockValidator(), persistDir: persistDir, } // Create the diffs for the genesis siafund outputs. for i, siafundOutput := range types.GenesisBlock.Transactions[0].SiafundOutputs { sfid := types.GenesisBlock.Transactions[0].SiafundOutputID(uint64(i)) sfod := modules.SiafundOutputDiff{ Direction: modules.DiffApply, ID: sfid, SiafundOutput: siafundOutput, } cs.blockRoot.SiafundOutputDiffs = append(cs.blockRoot.SiafundOutputDiffs, sfod) } // Initialize the consensus persistence structures. err := cs.initPersist() if err != nil { return nil, err } go func() { // Sync with the network. Don't sync if we are testing because // typically we don't have any mock peers to synchronize with in // testing. if bootstrap { // We are in a virgin goroutine right now, so calling the threaded // function without a goroutine is okay. err = cs.threadedInitialBlockchainDownload() if err != nil { return } } // threadedInitialBlockchainDownload will release the threadgroup 'Add' // it was holding, so another needs to be grabbed to finish off this // goroutine. err = cs.tg.Add() if err != nil { return } defer cs.tg.Done() // Register RPCs gateway.RegisterRPC("SendBlocks", cs.rpcSendBlocks) gateway.RegisterRPC("RelayBlock", cs.rpcRelayBlock) // COMPATv0.5.1 gateway.RegisterRPC("RelayHeader", cs.threadedRPCRelayHeader) gateway.RegisterRPC("SendBlk", cs.rpcSendBlk) gateway.RegisterConnectCall("SendBlocks", cs.threadedReceiveBlocks) cs.tg.OnStop(func() { cs.gateway.UnregisterRPC("SendBlocks") cs.gateway.UnregisterRPC("RelayBlock") cs.gateway.UnregisterRPC("RelayHeader") cs.gateway.UnregisterRPC("SendBlk") cs.gateway.UnregisterConnectCall("SendBlocks") }) // Mark that we are synced with the network. cs.mu.Lock() cs.synced = true cs.mu.Unlock() }() return cs, nil } // BlockAtHeight returns the block at a given height. func (cs *ConsensusSet) BlockAtHeight(height types.BlockHeight) (block types.Block, exists bool) { _ = cs.db.View(func(tx *bolt.Tx) error { id, err := getPath(tx, height) if err != nil { return err } pb, err := getBlockMap(tx, id) if err != nil { return err } block = pb.Block exists = true return nil }) return block, exists } // ChildTarget returns the target for the child of a block. func (cs *ConsensusSet) ChildTarget(id types.BlockID) (target types.Target, exists bool) { // A call to a closed database can cause undefined behavior. err := cs.tg.Add() if err != nil { return types.Target{}, false } defer cs.tg.Done() _ = cs.db.View(func(tx *bolt.Tx) error { pb, err := getBlockMap(tx, id) if err != nil { return err } target = pb.ChildTarget exists = true return nil }) return target, exists } // Close safely closes the block database. func (cs *ConsensusSet) Close() error { return cs.tg.Stop() } // managedCurrentBlock returns the latest block in the heaviest known blockchain. func (cs *ConsensusSet) managedCurrentBlock() (block types.Block) { cs.mu.RLock() defer cs.mu.RUnlock() _ = cs.db.View(func(tx *bolt.Tx) error { pb := currentProcessedBlock(tx) block = pb.Block return nil }) return block } // CurrentBlock returns the latest block in the heaviest known blockchain. func (cs *ConsensusSet) CurrentBlock() (block types.Block) { // A call to a closed database can cause undefined behavior. err := cs.tg.Add() if err != nil { return types.Block{} } defer cs.tg.Done() cs.mu.RLock() defer cs.mu.RUnlock() _ = cs.db.View(func(tx *bolt.Tx) error { pb := currentProcessedBlock(tx) block = pb.Block return nil }) return block } // Flush will block until the consensus set has finished all in-progress // routines. func (cs *ConsensusSet) Flush() error { return cs.tg.Flush() } // Height returns the height of the consensus set. func (cs *ConsensusSet) Height() (height types.BlockHeight) { // A call to a closed database can cause undefined behavior. err := cs.tg.Add() if err != nil { return 0 } defer cs.tg.Done() _ = cs.db.View(func(tx *bolt.Tx) error { height = blockHeight(tx) return nil }) return height } // InCurrentPath returns true if the block presented is in the current path, // false otherwise. func (cs *ConsensusSet) InCurrentPath(id types.BlockID) (inPath bool) { // A call to a closed database can cause undefined behavior. err := cs.tg.Add() if err != nil { return false } defer cs.tg.Done() _ = cs.db.View(func(tx *bolt.Tx) error { pb, err := getBlockMap(tx, id) if err != nil { inPath = false return nil } pathID, err := getPath(tx, pb.Height) if err != nil { inPath = false return nil } inPath = pathID == id return nil }) return inPath } // MinimumValidChildTimestamp returns the earliest timestamp that the next block // can have in order for it to be considered valid. func (cs *ConsensusSet) MinimumValidChildTimestamp(id types.BlockID) (timestamp types.Timestamp, exists bool) { // A call to a closed database can cause undefined behavior. err := cs.tg.Add() if err != nil { return 0, false } defer cs.tg.Done() // Error is not checked because it does not matter. _ = cs.db.View(func(tx *bolt.Tx) error { pb, err := getBlockMap(tx, id) if err != nil { return err } timestamp = cs.blockRuleHelper.minimumValidChildTimestamp(tx.Bucket(BlockMap), pb) exists = true return nil }) return timestamp, exists } // StorageProofSegment returns the segment to be used in the storage proof for // a given file contract. func (cs *ConsensusSet) StorageProofSegment(fcid types.FileContractID) (index uint64, err error) { // A call to a closed database can cause undefined behavior. err = cs.tg.Add() if err != nil { return 0, err } defer cs.tg.Done() _ = cs.db.View(func(tx *bolt.Tx) error { index, err = storageProofSegment(tx, fcid) return nil }) return index, err } Sia-1.0.4/modules/consensus/consensusset_bench_test.go000066400000000000000000000011101302033070500231630ustar00rootroot00000000000000package consensus import ( "strconv" "testing" ) // BenchmarkCreateServerTester benchmarks creating a server tester from // scratch. The consensus package creates over 60 server testers (and // counting), and optimizations to the server tester creation process are // likely to generalize to the project as a whole. // // i7-4770, 1d60d69: 22.883 ms / op func BenchmarkCreateServerTester(b *testing.B) { for i := 0; i < b.N; i++ { cst, err := createConsensusSetTester("BenchmarkCreateServerTester - " + strconv.Itoa(i)) if err != nil { b.Fatal(err) } cst.Close() } } Sia-1.0.4/modules/consensus/consensusset_test.go000066400000000000000000000121241302033070500220330ustar00rootroot00000000000000package consensus import ( "crypto/rand" "path/filepath" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/modules/miner" "github.com/NebulousLabs/Sia/modules/transactionpool" "github.com/NebulousLabs/Sia/modules/wallet" "github.com/NebulousLabs/Sia/types" ) // A consensusSetTester is the helper object for consensus set testing, // including helper modules and methods for controlling synchronization between // the tester and the modules. type consensusSetTester struct { gateway modules.Gateway miner modules.TestMiner tpool modules.TransactionPool wallet modules.Wallet walletKey crypto.TwofishKey cs *ConsensusSet persistDir string } // randAddress returns a random address that is not spendable. func randAddress() types.UnlockHash { var uh types.UnlockHash _, err := rand.Read(uh[:]) if err != nil { panic(err) } return uh } // addSiafunds makes a transaction that moves some testing genesis siafunds // into the wallet. func (cst *consensusSetTester) addSiafunds() { // Get an address to receive the siafunds. uc, err := cst.wallet.NextAddress() if err != nil { panic(err) } // Create the transaction that sends the anyone-can-spend siafund output to // the wallet address (output only available during testing). txn := types.Transaction{ SiafundInputs: []types.SiafundInput{{ ParentID: cst.cs.blockRoot.Block.Transactions[0].SiafundOutputID(2), UnlockConditions: types.UnlockConditions{}, }}, SiafundOutputs: []types.SiafundOutput{{ Value: types.NewCurrency64(1e3), UnlockHash: uc.UnlockHash(), }}, } // Mine the transaction into the blockchain. err = cst.tpool.AcceptTransactionSet([]types.Transaction{txn}) if err != nil { panic(err) } _, err = cst.miner.AddBlock() if err != nil { panic(err) } // Check that the siafunds made it to the wallet. _, siafundBalance, _ := cst.wallet.ConfirmedBalance() if !siafundBalance.Equals64(1e3) { panic("wallet does not have the siafunds") } } // mineCoins mines blocks until there are siacoins in the wallet. func (cst *consensusSetTester) mineSiacoins() { for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ { b, _ := cst.miner.FindBlock() err := cst.cs.AcceptBlock(b) if err != nil { panic(err) } } } // blankConsensusSetTester creates a consensusSetTester that has only the // genesis block. func blankConsensusSetTester(name string) (*consensusSetTester, error) { testdir := build.TempDir(modules.ConsensusDir, name) // Create modules. g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, err } cs, err := New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } key, err := crypto.GenerateTwofishKey() if err != nil { return nil, err } _, err = w.Encrypt(key) if err != nil { return nil, err } err = w.Unlock(key) if err != nil { return nil, err } m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.MinerDir)) if err != nil { return nil, err } // Assemble all objects into a consensusSetTester. cst := &consensusSetTester{ gateway: g, miner: m, tpool: tp, wallet: w, walletKey: key, cs: cs, persistDir: testdir, } return cst, nil } // createConsensusSetTester creates a consensusSetTester that's ready for use, // including siacoins and siafunds available in the wallet. func createConsensusSetTester(name string) (*consensusSetTester, error) { cst, err := blankConsensusSetTester(name) if err != nil { return nil, err } cst.addSiafunds() cst.mineSiacoins() return cst, nil } // Close safely closes the consensus set tester. Because there's not a good way // to errcheck when deferring a close, a panic is called in the event of an // error. func (cst *consensusSetTester) Close() error { errs := []error{ cst.cs.Close(), cst.gateway.Close(), cst.miner.Close(), } if err := build.JoinErrors(errs, "; "); err != nil { panic(err) } return nil } // TestNilInputs tries to create new consensus set modules using nil inputs. func TestNilInputs(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() testdir := build.TempDir(modules.ConsensusDir, "TestNilInputs") _, err := New(nil, false, testdir) if err != errNilGateway { t.Fatal(err) } } // TestClosing tries to close a consenuss set. func TestDatabaseClosing(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() testdir := build.TempDir(modules.ConsensusDir, "TestClosing") // Create the gateway. g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { t.Fatal(err) } cs, err := New(g, false, testdir) if err != nil { t.Fatal(err) } err = cs.Close() if err != nil { t.Error(err) } } Sia-1.0.4/modules/consensus/consistency.go000066400000000000000000000240071302033070500206040ustar00rootroot00000000000000package consensus import ( "bytes" "errors" "fmt" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // manageErr handles an error detected by the consistency checks. func manageErr(tx *bolt.Tx, err error) { markInconsistency(tx) if build.DEBUG { panic(err) } else { fmt.Println(err) } } // consensusChecksum grabs a checksum of the consensus set by pushing all of // the elements in sorted order into a merkle tree and taking the root. All // consensus sets with the same current block should have identical consensus // checksums. func consensusChecksum(tx *bolt.Tx) crypto.Hash { // Create a checksum tree. tree := crypto.NewTree() // For all of the constant buckets, push every key and every value. Buckets // are sorted in byte-order, therefore this operation is deterministic. consensusSetBuckets := []*bolt.Bucket{ tx.Bucket(BlockPath), tx.Bucket(SiacoinOutputs), tx.Bucket(FileContracts), tx.Bucket(SiafundOutputs), tx.Bucket(SiafundPool), } for i := range consensusSetBuckets { err := consensusSetBuckets[i].ForEach(func(k, v []byte) error { tree.Push(k) tree.Push(v) return nil }) if err != nil { manageErr(tx, err) } } // Iterate through all the buckets looking for buckets prefixed with // prefixDSCO or prefixFCEX. Buckets are presented in byte-sorted order by // name. err := tx.ForEach(func(name []byte, b *bolt.Bucket) error { // If the bucket is not a delayed siacoin output bucket or a file // contract expiration bucket, skip. if !bytes.HasPrefix(name, prefixDSCO) && !bytes.HasPrefix(name, prefixFCEX) { return nil } // The bucket is a prefixed bucket - add all elements to the tree. return b.ForEach(func(k, v []byte) error { tree.Push(k) tree.Push(v) return nil }) }) if err != nil { manageErr(tx, err) } return tree.Root() } // checkSiacoinCount checks that the number of siacoins countable within the // consensus set equal the expected number of siacoins for the block height. func checkSiacoinCount(tx *bolt.Tx) { // Iterate through all the buckets looking for the delayed siacoin output // buckets, and check that they are for the correct heights. var dscoSiacoins types.Currency err := tx.ForEach(func(name []byte, b *bolt.Bucket) error { // Check if the bucket is a delayed siacoin output bucket. if !bytes.HasPrefix(name, prefixDSCO) { return nil } // Sum up the delayed outputs in this bucket. err := b.ForEach(func(_, delayedOutput []byte) error { var sco types.SiacoinOutput err := encoding.Unmarshal(delayedOutput, &sco) if err != nil { manageErr(tx, err) } dscoSiacoins = dscoSiacoins.Add(sco.Value) return nil }) if err != nil { return err } return nil }) if err != nil { manageErr(tx, err) } // Add all of the siacoin outputs. var scoSiacoins types.Currency err = tx.Bucket(SiacoinOutputs).ForEach(func(_, scoBytes []byte) error { var sco types.SiacoinOutput err := encoding.Unmarshal(scoBytes, &sco) if err != nil { manageErr(tx, err) } scoSiacoins = scoSiacoins.Add(sco.Value) return nil }) if err != nil { manageErr(tx, err) } // Add all of the payouts from file contracts. var fcSiacoins types.Currency err = tx.Bucket(FileContracts).ForEach(func(_, fcBytes []byte) error { var fc types.FileContract err := encoding.Unmarshal(fcBytes, &fc) if err != nil { manageErr(tx, err) } var fcCoins types.Currency for _, output := range fc.ValidProofOutputs { fcCoins = fcCoins.Add(output.Value) } fcSiacoins = fcSiacoins.Add(fcCoins) return nil }) if err != nil { manageErr(tx, err) } // Add all of the siafund claims. var claimSiacoins types.Currency err = tx.Bucket(SiafundOutputs).ForEach(func(_, sfoBytes []byte) error { var sfo types.SiafundOutput err := encoding.Unmarshal(sfoBytes, &sfo) if err != nil { manageErr(tx, err) } coinsPerFund := getSiafundPool(tx).Sub(sfo.ClaimStart) claimCoins := coinsPerFund.Mul(sfo.Value).Div(types.SiafundCount) claimSiacoins = claimSiacoins.Add(claimCoins) return nil }) if err != nil { manageErr(tx, err) } expectedSiacoins := types.CalculateNumSiacoins(blockHeight(tx)) totalSiacoins := dscoSiacoins.Add(scoSiacoins).Add(fcSiacoins).Add(claimSiacoins) if !totalSiacoins.Equals(expectedSiacoins) { diagnostics := fmt.Sprintf("Wrong number of siacoins\nDsco: %v\nSco: %v\nFc: %v\nClaim: %v\n", dscoSiacoins, scoSiacoins, fcSiacoins, claimSiacoins) if totalSiacoins.Cmp(expectedSiacoins) < 0 { diagnostics += fmt.Sprintf("total: %v\nexpected: %v\n expected is bigger: %v", totalSiacoins, expectedSiacoins, expectedSiacoins.Sub(totalSiacoins)) } else { diagnostics += fmt.Sprintf("total: %v\nexpected: %v\n expected is bigger: %v", totalSiacoins, expectedSiacoins, totalSiacoins.Sub(expectedSiacoins)) } manageErr(tx, errors.New(diagnostics)) } } // checkSiafundCount checks that the number of siafunds countable within the // consensus set equal the expected number of siafunds for the block height. func checkSiafundCount(tx *bolt.Tx) { var total types.Currency err := tx.Bucket(SiafundOutputs).ForEach(func(_, siafundOutputBytes []byte) error { var sfo types.SiafundOutput err := encoding.Unmarshal(siafundOutputBytes, &sfo) if err != nil { manageErr(tx, err) } total = total.Add(sfo.Value) return nil }) if err != nil { manageErr(tx, err) } if !total.Equals(types.SiafundCount) { manageErr(tx, errors.New("wrong number if siafunds in the consensus set")) } } // checkDSCOs scans the sets of delayed siacoin outputs and checks for // consistency. func checkDSCOs(tx *bolt.Tx) { // Create a map to track which delayed siacoin output maps exist, and // another map to track which ids have appeared in the dsco set. dscoTracker := make(map[types.BlockHeight]struct{}) idMap := make(map[types.SiacoinOutputID]struct{}) // Iterate through all the buckets looking for the delayed siacoin output // buckets, and check that they are for the correct heights. err := tx.ForEach(func(name []byte, b *bolt.Bucket) error { // If the bucket is not a delayed siacoin output bucket or a file // contract expiration bucket, skip. if !bytes.HasPrefix(name, prefixDSCO) { return nil } // Add the bucket to the dscoTracker. var height types.BlockHeight err := encoding.Unmarshal(name[len(prefixDSCO):], &height) if err != nil { manageErr(tx, err) } _, exists := dscoTracker[height] if exists { return errors.New("repeat dsco map") } dscoTracker[height] = struct{}{} var total types.Currency err = b.ForEach(func(idBytes, delayedOutput []byte) error { // Check that the output id has not appeared in another dsco. var id types.SiacoinOutputID copy(id[:], idBytes) _, exists := idMap[id] if exists { return errors.New("repeat delayed siacoin output") } idMap[id] = struct{}{} // Sum the funds in the bucket. var sco types.SiacoinOutput err := encoding.Unmarshal(delayedOutput, &sco) if err != nil { manageErr(tx, err) } total = total.Add(sco.Value) return nil }) if err != nil { return err } // Check that the minimum value has been achieved - the coinbase from // an earlier block is guaranteed to be in the bucket. minimumValue := types.CalculateCoinbase(height - types.MaturityDelay) if total.Cmp(minimumValue) < 0 { return errors.New("total number of coins in the delayed output bucket is incorrect") } return nil }) if err != nil { manageErr(tx, err) } // Check that all of the correct heights are represented. currentHeight := blockHeight(tx) expectedBuckets := 0 for i := currentHeight + 1; i <= currentHeight+types.MaturityDelay; i++ { if i < types.MaturityDelay { continue } _, exists := dscoTracker[i] if !exists { manageErr(tx, errors.New("missing a dsco bucket")) } expectedBuckets++ } if len(dscoTracker) != expectedBuckets { manageErr(tx, errors.New("too many dsco buckets")) } } // checkRevertApply reverts the most recent block, checking to see that the // consensus set hash matches the hash obtained for the previous block. Then it // applies the block again and checks that the consensus set hash matches the // original consensus set hash. func (cs *ConsensusSet) checkRevertApply(tx *bolt.Tx) { current := currentProcessedBlock(tx) // Don't perform the check if this block is the genesis block. if current.Block.ID() == cs.blockRoot.Block.ID() { return } parent, err := getBlockMap(tx, current.Block.ParentID) if err != nil { manageErr(tx, err) } if current.Height != parent.Height+1 { manageErr(tx, errors.New("parent structure of a block is incorrect")) } _, _, err = cs.forkBlockchain(tx, parent) if err != nil { manageErr(tx, err) } if consensusChecksum(tx) != parent.ConsensusChecksum { manageErr(tx, errors.New("consensus checksum mismatch after reverting")) } _, _, err = cs.forkBlockchain(tx, current) if err != nil { manageErr(tx, err) } if consensusChecksum(tx) != current.ConsensusChecksum { manageErr(tx, errors.New("consensus checksum mismatch after re-applying")) } } // checkConsistency runs a series of checks to make sure that the consensus set // is consistent with some rules that should always be true. func (cs *ConsensusSet) checkConsistency(tx *bolt.Tx) { if cs.checkingConsistency { return } cs.checkingConsistency = true checkDSCOs(tx) checkSiacoinCount(tx) checkSiafundCount(tx) if build.DEBUG { cs.checkRevertApply(tx) } cs.checkingConsistency = false } // maybeCheckConsistency runs a consistency check with a small probability. // Useful for detecting database corruption in production without needing to go // through the extremely slow process of running a consistency check every // block. func (cs *ConsensusSet) maybeCheckConsistency(tx *bolt.Tx) { n, err := crypto.RandIntn(1000) if err != nil { manageErr(tx, err) } if n == 0 { cs.checkConsistency(tx) } } // TODO: Check that every file contract has an expiration too, and that the // number of file contracts + the number of expirations is equal. Sia-1.0.4/modules/consensus/consistency_helpers_test.go000066400000000000000000000006451302033070500233670ustar00rootroot00000000000000package consensus import ( "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/bolt" ) // dbConsensusChecksum is a convenience function to call consensusChecksum // without a bolt.Tx. func (cs *ConsensusSet) dbConsensusChecksum() (checksum crypto.Hash) { err := cs.db.Update(func(tx *bolt.Tx) error { checksum = consensusChecksum(tx) return nil }) if err != nil { panic(err) } return checksum } Sia-1.0.4/modules/consensus/database.go000066400000000000000000000102121302033070500200000ustar00rootroot00000000000000package consensus // database.go contains functions to initialize the database and report // inconsistencies. All of the database-specific logic belongs here. import ( "errors" "fmt" "os" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/bolt" ) var ( errRepeatInsert = errors.New("attempting to add an already existing item to the consensus set") errNilBucket = errors.New("using a bucket that does not exist") errNilItem = errors.New("requested item does not exist") errDBInconsistent = errors.New("database guard indicates inconsistency within database") errNonEmptyBucket = errors.New("cannot remove a map with objects still in it") dbMetadata = persist.Metadata{ Header: "Consensus Set Database", Version: "0.5.0", } ) type ( // dbBucket represents a collection of key/value pairs inside the database. dbBucket interface { Get(key []byte) []byte } // dbTx represents a read-only transaction on the database that can be used // for retrieving values. dbTx interface { Bucket(name []byte) dbBucket } // boltTxWrapper wraps a bolt.Tx so that it matches the dbTx interface. The // wrap is necessary because bolt.Tx.Bucket() returns a fixed type // (bolt.Bucket), but we want it to return an interface (dbBucket). boltTxWrapper struct { tx *bolt.Tx } ) // Bucket returns the dbBucket associated with the given bucket name. func (b boltTxWrapper) Bucket(name []byte) dbBucket { return b.tx.Bucket(name) } // replaceDatabase backs up the existing database and creates a new one. func (cs *ConsensusSet) replaceDatabase(filename string) error { // Rename the existing database and create a new one. fmt.Println("Outdated consensus database... backing up and replacing") err := os.Rename(filename, filename+".bck") if err != nil { return errors.New("error while backing up consensus database: " + err.Error()) } // Try again to create a new database, this time without checking for an // outdated database error. cs.db, err = persist.OpenDatabase(dbMetadata, filename) if err != nil { return errors.New("error opening consensus database: " + err.Error()) } return nil } // openDB loads the set database and populates it with the necessary buckets func (cs *ConsensusSet) openDB(filename string) (err error) { cs.db, err = persist.OpenDatabase(dbMetadata, filename) if err == persist.ErrBadVersion { return cs.replaceDatabase(filename) } if err != nil { return errors.New("error opening consensus database: " + err.Error()) } return nil } // dbInitialized returns true if the database appears to be initialized, false // if not. Checking for the existence of the siafund pool bucket is typically // sufficient to determine whether the database has gone through the // initialization process. func dbInitialized(tx *bolt.Tx) bool { return tx.Bucket(SiafundPool) != nil } // initDB is run if there is no existing consensus database, creating a // database with all the required buckets and sane initial values. func (cs *ConsensusSet) initDB(tx *bolt.Tx) error { // Create the compononents of the database. err := cs.createConsensusDB(tx) if err != nil { return err } err = cs.createChangeLog(tx) if err != nil { return err } // Place a 'false' in the consistency bucket to indicate that no // inconsistencies have been found. err = tx.Bucket(Consistency).Put(Consistency, encoding.Marshal(false)) if err != nil { return err } return nil } // inconsistencyDetected indicates whether inconsistency has been detected // within the database. func inconsistencyDetected(tx *bolt.Tx) (detected bool) { inconsistencyBytes := tx.Bucket(Consistency).Get(Consistency) err := encoding.Unmarshal(inconsistencyBytes, &detected) if build.DEBUG && err != nil { panic(err) } return detected } // markInconsistency flags the database to indicate that inconsistency has been // detected. func markInconsistency(tx *bolt.Tx) { // Place a 'true' in the consistency bucket to indicate that // inconsistencies have been found. err := tx.Bucket(Consistency).Put(Consistency, encoding.Marshal(true)) if build.DEBUG && err != nil { panic(err) } } Sia-1.0.4/modules/consensus/diffs.go000066400000000000000000000223771302033070500173460ustar00rootroot00000000000000package consensus import ( "errors" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) var ( errApplySiafundPoolDiffMismatch = errors.New("committing a siafund pool diff with an invalid 'previous' field") errDiffsNotGenerated = errors.New("applying diff set before generating errors") errInvalidSuccessor = errors.New("generating diffs for a block that's an invalid successsor to the current block") errNegativePoolAdjustment = errors.New("committing a siafund pool diff with a negative adjustment") errNonApplySiafundPoolDiff = errors.New("commiting a siafund pool diff that doesn't have the 'apply' direction") errRevertSiafundPoolDiffMismatch = errors.New("committing a siafund pool diff with an invalid 'adjusted' field") errWrongAppliedDiffSet = errors.New("applying a diff set that isn't the current block") errWrongRevertDiffSet = errors.New("reverting a diff set that isn't the current block") ) // commitDiffSetSanity performs a series of sanity checks before committing a // diff set. func commitDiffSetSanity(tx *bolt.Tx, pb *processedBlock, dir modules.DiffDirection) { // This function is purely sanity checks. if !build.DEBUG { return } // Diffs should have already been generated for this node. if !pb.DiffsGenerated { panic(errDiffsNotGenerated) } // Current node must be the input node's parent if applying, and // current node must be the input node if reverting. if dir == modules.DiffApply { parent, err := getBlockMap(tx, pb.Block.ParentID) if build.DEBUG && err != nil { panic(err) } if parent.Block.ID() != currentBlockID(tx) { panic(errWrongAppliedDiffSet) } } else { if pb.Block.ID() != currentBlockID(tx) { panic(errWrongRevertDiffSet) } } } // commitSiacoinOutputDiff applies or reverts a SiacoinOutputDiff. func commitSiacoinOutputDiff(tx *bolt.Tx, scod modules.SiacoinOutputDiff, dir modules.DiffDirection) { if scod.Direction == dir { addSiacoinOutput(tx, scod.ID, scod.SiacoinOutput) } else { removeSiacoinOutput(tx, scod.ID) } } // commitFileContractDiff applies or reverts a FileContractDiff. func commitFileContractDiff(tx *bolt.Tx, fcd modules.FileContractDiff, dir modules.DiffDirection) { if fcd.Direction == dir { addFileContract(tx, fcd.ID, fcd.FileContract) } else { removeFileContract(tx, fcd.ID) } } // commitSiafundOutputDiff applies or reverts a Siafund output diff. func commitSiafundOutputDiff(tx *bolt.Tx, sfod modules.SiafundOutputDiff, dir modules.DiffDirection) { if sfod.Direction == dir { addSiafundOutput(tx, sfod.ID, sfod.SiafundOutput) } else { removeSiafundOutput(tx, sfod.ID) } } // commitDelayedSiacoinOutputDiff applies or reverts a delayedSiacoinOutputDiff. func commitDelayedSiacoinOutputDiff(tx *bolt.Tx, dscod modules.DelayedSiacoinOutputDiff, dir modules.DiffDirection) { if dscod.Direction == dir { addDSCO(tx, dscod.MaturityHeight, dscod.ID, dscod.SiacoinOutput) } else { removeDSCO(tx, dscod.MaturityHeight, dscod.ID) } } // commitSiafundPoolDiff applies or reverts a SiafundPoolDiff. func commitSiafundPoolDiff(tx *bolt.Tx, sfpd modules.SiafundPoolDiff, dir modules.DiffDirection) { // Sanity check - siafund pool should only ever increase. if build.DEBUG { if sfpd.Adjusted.Cmp(sfpd.Previous) < 0 { panic(errNegativePoolAdjustment) } if sfpd.Direction != modules.DiffApply { panic(errNonApplySiafundPoolDiff) } } if dir == modules.DiffApply { // Sanity check - sfpd.Previous should equal the current siafund pool. if build.DEBUG && !getSiafundPool(tx).Equals(sfpd.Previous) { panic(errApplySiafundPoolDiffMismatch) } setSiafundPool(tx, sfpd.Adjusted) } else { // Sanity check - sfpd.Adjusted should equal the current siafund pool. if build.DEBUG && !getSiafundPool(tx).Equals(sfpd.Adjusted) { panic(errRevertSiafundPoolDiffMismatch) } setSiafundPool(tx, sfpd.Previous) } } // createUpcomingDelayeOutputdMaps creates the delayed siacoin output maps that // will be used when applying delayed siacoin outputs in the diff set. func createUpcomingDelayedOutputMaps(tx *bolt.Tx, pb *processedBlock, dir modules.DiffDirection) { if dir == modules.DiffApply { createDSCOBucket(tx, pb.Height+types.MaturityDelay) } else if pb.Height >= types.MaturityDelay { createDSCOBucket(tx, pb.Height) } } // commitNodeDiffs commits all of the diffs in a block node. func commitNodeDiffs(tx *bolt.Tx, pb *processedBlock, dir modules.DiffDirection) { if dir == modules.DiffApply { for _, scod := range pb.SiacoinOutputDiffs { commitSiacoinOutputDiff(tx, scod, dir) } for _, fcd := range pb.FileContractDiffs { commitFileContractDiff(tx, fcd, dir) } for _, sfod := range pb.SiafundOutputDiffs { commitSiafundOutputDiff(tx, sfod, dir) } for _, dscod := range pb.DelayedSiacoinOutputDiffs { commitDelayedSiacoinOutputDiff(tx, dscod, dir) } for _, sfpd := range pb.SiafundPoolDiffs { commitSiafundPoolDiff(tx, sfpd, dir) } } else { for i := len(pb.SiacoinOutputDiffs) - 1; i >= 0; i-- { commitSiacoinOutputDiff(tx, pb.SiacoinOutputDiffs[i], dir) } for i := len(pb.FileContractDiffs) - 1; i >= 0; i-- { commitFileContractDiff(tx, pb.FileContractDiffs[i], dir) } for i := len(pb.SiafundOutputDiffs) - 1; i >= 0; i-- { commitSiafundOutputDiff(tx, pb.SiafundOutputDiffs[i], dir) } for i := len(pb.DelayedSiacoinOutputDiffs) - 1; i >= 0; i-- { commitDelayedSiacoinOutputDiff(tx, pb.DelayedSiacoinOutputDiffs[i], dir) } for i := len(pb.SiafundPoolDiffs) - 1; i >= 0; i-- { commitSiafundPoolDiff(tx, pb.SiafundPoolDiffs[i], dir) } } } // deleteObsoleteDelayedOutputMaps deletes the delayed siacoin output maps that // are no longer in use. func deleteObsoleteDelayedOutputMaps(tx *bolt.Tx, pb *processedBlock, dir modules.DiffDirection) { // There are no outputs that mature in the first MaturityDelay blocks. if dir == modules.DiffApply && pb.Height >= types.MaturityDelay { deleteDSCOBucket(tx, pb.Height) } else if dir == modules.DiffRevert { deleteDSCOBucket(tx, pb.Height+types.MaturityDelay) } } // updateCurrentPath updates the current path after applying a diff set. func updateCurrentPath(tx *bolt.Tx, pb *processedBlock, dir modules.DiffDirection) { // Update the current path. if dir == modules.DiffApply { pushPath(tx, pb.Block.ID()) } else { popPath(tx) } } // commitDiffSet applies or reverts the diffs in a blockNode. func commitDiffSet(tx *bolt.Tx, pb *processedBlock, dir modules.DiffDirection) { // Sanity checks - there are a few so they were moved to another function. if build.DEBUG { commitDiffSetSanity(tx, pb, dir) } createUpcomingDelayedOutputMaps(tx, pb, dir) commitNodeDiffs(tx, pb, dir) deleteObsoleteDelayedOutputMaps(tx, pb, dir) updateCurrentPath(tx, pb, dir) } // generateAndApplyDiff will verify the block and then integrate it into the // consensus state. These two actions must happen at the same time because // transactions are allowed to depend on each other. We can't be sure that a // transaction is valid unless we have applied all of the previous transactions // in the block, which means we need to apply while we verify. func generateAndApplyDiff(tx *bolt.Tx, pb *processedBlock) error { // Sanity check - the block being applied should have the current block as // a parent. if build.DEBUG && pb.Block.ParentID != currentBlockID(tx) { panic(errInvalidSuccessor) } // Create the bucket to hold all of the delayed siacoin outputs created by // transactions this block. Needs to happen before any transactions are // applied. createDSCOBucket(tx, pb.Height+types.MaturityDelay) // Validate and apply each transaction in the block. They cannot be // validated all at once because some transactions may not be valid until // previous transactions have been applied. for _, txn := range pb.Block.Transactions { err := validTransaction(tx, txn) if err != nil { return err } applyTransaction(tx, pb, txn) } // After all of the transactions have been applied, 'maintenance' is // applied on the block. This includes adding any outputs that have reached // maturity, applying any contracts with missed storage proofs, and adding // the miner payouts to the list of delayed outputs. applyMaintenance(tx, pb) // DiffsGenerated are only set to true after the block has been fully // validated and integrated. This is required to prevent later blocks from // being accepted on top of an invalid block - if the consensus set ever // forks over an invalid block, 'DiffsGenerated' will be set to 'false', // requiring validation to occur again. when 'DiffsGenerated' is set to // true, validation is skipped, therefore the flag should only be set to // true on fully validated blocks. pb.DiffsGenerated = true // Add the block to the current path and block map. bid := pb.Block.ID() blockMap := tx.Bucket(BlockMap) updateCurrentPath(tx, pb, modules.DiffApply) // Sanity check preparation - set the consensus hash at this height so that // during reverting a check can be performed to assure consistency when // adding and removing blocks. Must happen after the block is added to the // path. if build.DEBUG { pb.ConsensusChecksum = consensusChecksum(tx) } return blockMap.Put(bid[:], encoding.Marshal(*pb)) } Sia-1.0.4/modules/consensus/diffs_test.go000066400000000000000000000500131302033070500203710ustar00rootroot00000000000000package consensus import ( "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // TestCommitDelayedSiacoinOutputDiffBadMaturity commits a delayed siacoin // output that has a bad maturity height and triggers a panic. func TestCommitDelayedSiacoinOutputDiffBadMaturity(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestCommitDelayedSiacoinOutputDiffBadMaturity") if err != nil { t.Fatal(err) } defer cst.Close() // Trigger an inconsistency check. defer func() { r := recover() if r == nil { t.Error("expecting error after corrupting database") } }() // Commit a delayed siacoin output with maturity height = cs.height()+1 maturityHeight := cst.cs.dbBlockHeight() - 1 id := types.SiacoinOutputID{'1'} dsco := types.SiacoinOutput{Value: types.NewCurrency64(1)} dscod := modules.DelayedSiacoinOutputDiff{ Direction: modules.DiffApply, ID: id, SiacoinOutput: dsco, MaturityHeight: maturityHeight, } _ = cst.cs.db.Update(func(tx *bolt.Tx) error { commitDelayedSiacoinOutputDiff(tx, dscod, modules.DiffApply) return nil }) } // TestCommitNodeDiffs probes the commitNodeDiffs method of the consensus set. /* func TestCommitNodeDiffs(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestCommitNodeDiffs") if err != nil { t.Fatal(err) } defer cst.Close() pb := cst.cs.dbCurrentProcessedBlock() _ = cst.cs.db.Update(func(tx *bolt.Tx) error { commitDiffSet(tx, pb, modules.DiffRevert) // pull the block node out of the consensus set. return nil }) // For diffs that can be destroyed in the same block they are created, // create diffs that do just that. This has in the past caused issues upon // rewinding. scoid := types.SiacoinOutputID{'1'} scod0 := modules.SiacoinOutputDiff{ Direction: modules.DiffApply, ID: scoid, } scod1 := modules.SiacoinOutputDiff{ Direction: modules.DiffRevert, ID: scoid, } fcid := types.FileContractID{'2'} fcd0 := modules.FileContractDiff{ Direction: modules.DiffApply, ID: fcid, } fcd1 := modules.FileContractDiff{ Direction: modules.DiffRevert, ID: fcid, } sfoid := types.SiafundOutputID{'3'} sfod0 := modules.SiafundOutputDiff{ Direction: modules.DiffApply, ID: sfoid, } sfod1 := modules.SiafundOutputDiff{ Direction: modules.DiffRevert, ID: sfoid, } dscoid := types.SiacoinOutputID{'4'} dscod := modules.DelayedSiacoinOutputDiff{ Direction: modules.DiffApply, ID: dscoid, MaturityHeight: cst.cs.dbBlockHeight() + types.MaturityDelay, } var siafundPool types.Currency err = cst.cs.db.Update(func(tx *bolt.Tx) error { siafundPool = getSiafundPool(tx) return nil }) if err != nil { panic(err) } sfpd := modules.SiafundPoolDiff{ Direction: modules.DiffApply, Previous: siafundPool, Adjusted: siafundPool.Add(types.NewCurrency64(1)), } pb.SiacoinOutputDiffs = append(pb.SiacoinOutputDiffs, scod0) pb.SiacoinOutputDiffs = append(pb.SiacoinOutputDiffs, scod1) pb.FileContractDiffs = append(pb.FileContractDiffs, fcd0) pb.FileContractDiffs = append(pb.FileContractDiffs, fcd1) pb.SiafundOutputDiffs = append(pb.SiafundOutputDiffs, sfod0) pb.SiafundOutputDiffs = append(pb.SiafundOutputDiffs, sfod1) pb.DelayedSiacoinOutputDiffs = append(pb.DelayedSiacoinOutputDiffs, dscod) pb.SiafundPoolDiffs = append(pb.SiafundPoolDiffs, sfpd) _ = cst.cs.db.Update(func(tx *bolt.Tx) error { createUpcomingDelayedOutputMaps(tx, pb, modules.DiffApply) return nil }) _ = cst.cs.db.Update(func(tx *bolt.Tx) error { commitNodeDiffs(tx, pb, modules.DiffApply) return nil }) exists := cst.cs.db.inSiacoinOutputs(scoid) if exists { t.Error("intradependent outputs not treated correctly") } exists = cst.cs.db.inFileContracts(fcid) if exists { t.Error("intradependent outputs not treated correctly") } exists = cst.cs.db.inSiafundOutputs(sfoid) if exists { t.Error("intradependent outputs not treated correctly") } _ = cst.cs.db.Update(func(tx *bolt.Tx) error { commitNodeDiffs(tx, pb, modules.DiffRevert) return nil }) exists = cst.cs.db.inSiacoinOutputs(scoid) if exists { t.Error("intradependent outputs not treated correctly") } exists = cst.cs.db.inFileContracts(fcid) if exists { t.Error("intradependent outputs not treated correctly") } exists = cst.cs.db.inSiafundOutputs(sfoid) if exists { t.Error("intradependent outputs not treated correctly") } } */ /* // TestSiacoinOutputDiff applies and reverts a siacoin output diff, then // triggers an inconsistency panic. func TestCommitSiacoinOutputDiff(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestCommitSiacoinOutputDiff") if err != nil { t.Fatal(err) } defer cst.Close() // Commit a siacoin output diff. initialScosLen := cst.cs.db.lenSiacoinOutputs() id := types.SiacoinOutputID{'1'} sco := types.SiacoinOutput{Value: types.NewCurrency64(1)} scod := modules.SiacoinOutputDiff{ Direction: modules.DiffApply, ID: id, SiacoinOutput: sco, } cst.cs.commitSiacoinOutputDiff(scod, modules.DiffApply) if cst.cs.db.lenSiacoinOutputs() != initialScosLen+1 { t.Error("siacoin output diff set did not increase in size") } if cst.cs.db.getSiacoinOutputs(id).Value.Cmp(sco.Value) != 0 { t.Error("wrong siacoin output value after committing a diff") } // Rewind the diff. cst.cs.commitSiacoinOutputDiff(scod, modules.DiffRevert) if cst.cs.db.lenSiacoinOutputs() != initialScosLen { t.Error("siacoin output diff set did not increase in size") } exists := cst.cs.db.inSiacoinOutputs(id) if exists { t.Error("siacoin output was not reverted") } // Restore the diff and then apply the inverse diff. cst.cs.commitSiacoinOutputDiff(scod, modules.DiffApply) scod.Direction = modules.DiffRevert cst.cs.commitSiacoinOutputDiff(scod, modules.DiffApply) if cst.cs.db.lenSiacoinOutputs() != initialScosLen { t.Error("siacoin output diff set did not increase in size") } exists = cst.cs.db.inSiacoinOutputs(id) if exists { t.Error("siacoin output was not reverted") } // Revert the inverse diff. cst.cs.commitSiacoinOutputDiff(scod, modules.DiffRevert) if cst.cs.db.lenSiacoinOutputs() != initialScosLen+1 { t.Error("siacoin output diff set did not increase in size") } if cst.cs.db.getSiacoinOutputs(id).Value.Cmp(sco.Value) != 0 { t.Error("wrong siacoin output value after committing a diff") } // Trigger an inconsistency check. defer func() { r := recover() if r != errBadCommitSiacoinOutputDiff { t.Error("expecting errBadCommitSiacoinOutputDiff, got", r) } }() // Try reverting a revert diff that was already reverted. (add an object // that already exists) cst.cs.commitSiacoinOutputDiff(scod, modules.DiffRevert) } */ /* // TestCommitFileContracttDiff applies and reverts a file contract diff, then // triggers an inconsistency panic. func TestCommitFileContractDiff(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestCommitFileContractDiff") if err != nil { t.Fatal(err) } // Commit a file contract diff. initialFcsLen := cst.cs.db.lenFileContracts() id := types.FileContractID{'1'} fc := types.FileContract{Payout: types.NewCurrency64(1)} fcd := modules.FileContractDiff{ Direction: modules.DiffApply, ID: id, FileContract: fc, } cst.cs.commitFileContractDiff(fcd, modules.DiffApply) if cst.cs.db.lenFileContracts() != initialFcsLen+1 { t.Error("siacoin output diff set did not increase in size") } if cst.cs.db.getFileContracts(id).Payout.Cmp(fc.Payout) != 0 { t.Error("wrong siacoin output value after committing a diff") } // Rewind the diff. cst.cs.commitFileContractDiff(fcd, modules.DiffRevert) if cst.cs.db.lenFileContracts() != initialFcsLen { t.Error("siacoin output diff set did not increase in size") } exists := cst.cs.db.inFileContracts(id) if exists { t.Error("siacoin output was not reverted") } // Restore the diff and then apply the inverse diff. cst.cs.commitFileContractDiff(fcd, modules.DiffApply) fcd.Direction = modules.DiffRevert cst.cs.commitFileContractDiff(fcd, modules.DiffApply) if cst.cs.db.lenFileContracts() != initialFcsLen { t.Error("siacoin output diff set did not increase in size") } exists = cst.cs.db.inFileContracts(id) if exists { t.Error("siacoin output was not reverted") } // Revert the inverse diff. cst.cs.commitFileContractDiff(fcd, modules.DiffRevert) if cst.cs.db.lenFileContracts() != initialFcsLen+1 { t.Error("siacoin output diff set did not increase in size") } if cst.cs.db.getFileContracts(id).Payout.Cmp(fc.Payout) != 0 { t.Error("wrong siacoin output value after committing a diff") } // Trigger an inconsistency check. defer func() { r := recover() if r != errBadCommitFileContractDiff { t.Error("expecting errBadCommitFileContractDiff, got", r) } }() // Try reverting an apply diff that was already reverted. (remove an object // that was already removed) fcd.Direction = modules.DiffApply // Object currently exists, but make the direction 'apply'. cst.cs.commitFileContractDiff(fcd, modules.DiffRevert) // revert the application. cst.cs.commitFileContractDiff(fcd, modules.DiffRevert) // revert the application again, in error. } */ // TestSiafundOutputDiff applies and reverts a siafund output diff, then // triggers an inconsistency panic. /* func TestCommitSiafundOutputDiff(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestCommitSiafundOutputDiff") if err != nil { t.Fatal(err) } // Commit a siafund output diff. initialScosLen := cst.cs.db.lenSiafundOutputs() id := types.SiafundOutputID{'1'} sfo := types.SiafundOutput{Value: types.NewCurrency64(1)} sfod := modules.SiafundOutputDiff{ Direction: modules.DiffApply, ID: id, SiafundOutput: sfo, } cst.cs.commitSiafundOutputDiff(sfod, modules.DiffApply) if cst.cs.db.lenSiafundOutputs() != initialScosLen+1 { t.Error("siafund output diff set did not increase in size") } sfo1 := cst.cs.db.getSiafundOutputs(id) if sfo1.Value.Cmp(sfo.Value) != 0 { t.Error("wrong siafund output value after committing a diff") } // Rewind the diff. cst.cs.commitSiafundOutputDiff(sfod, modules.DiffRevert) if cst.cs.db.lenSiafundOutputs() != initialScosLen { t.Error("siafund output diff set did not increase in size") } exists := cst.cs.db.inSiafundOutputs(id) if exists { t.Error("siafund output was not reverted") } // Restore the diff and then apply the inverse diff. cst.cs.commitSiafundOutputDiff(sfod, modules.DiffApply) sfod.Direction = modules.DiffRevert cst.cs.commitSiafundOutputDiff(sfod, modules.DiffApply) if cst.cs.db.lenSiafundOutputs() != initialScosLen { t.Error("siafund output diff set did not increase in size") } exists = cst.cs.db.inSiafundOutputs(id) if exists { t.Error("siafund output was not reverted") } // Revert the inverse diff. cst.cs.commitSiafundOutputDiff(sfod, modules.DiffRevert) if cst.cs.db.lenSiafundOutputs() != initialScosLen+1 { t.Error("siafund output diff set did not increase in size") } sfo2 := cst.cs.db.getSiafundOutputs(id) if sfo2.Value.Cmp(sfo.Value) != 0 { t.Error("wrong siafund output value after committing a diff") } // Trigger an inconsistency check. defer func() { r := recover() if r != errBadCommitSiafundOutputDiff { t.Error("expecting errBadCommitSiafundOutputDiff, got", r) } }() // Try applying a revert diff that was already applied. (remove an object // that was already removed) cst.cs.commitSiafundOutputDiff(sfod, modules.DiffApply) // Remove the object. cst.cs.commitSiafundOutputDiff(sfod, modules.DiffApply) // Remove the object again. } */ // TestCommitDelayedSiacoinOutputDiff probes the commitDelayedSiacoinOutputDiff // method of the consensus set. /* func TestCommitDelayedSiacoinOutputDiff(t *testing.T) { t.Skip("test isn't working, but checks the wrong code anyway") if testing.Short() { t.Skip() } cst, err := createConsensusSetTester("TestCommitDelayedSiacoinOutputDiff") if err != nil { t.Fatal(err) } // Commit a delayed siacoin output with maturity height = cs.height()+1 maturityHeight := cst.cs.height() + 1 initialDscosLen := cst.cs.db.lenDelayedSiacoinOutputsHeight(maturityHeight) id := types.SiacoinOutputID{'1'} dsco := types.SiacoinOutput{Value: types.NewCurrency64(1)} dscod := modules.DelayedSiacoinOutputDiff{ Direction: modules.DiffApply, ID: id, SiacoinOutput: dsco, MaturityHeight: maturityHeight, } cst.cs.commitDelayedSiacoinOutputDiff(dscod, modules.DiffApply) if cst.cs.db.lenDelayedSiacoinOutputsHeight(maturityHeight) != initialDscosLen+1 { t.Fatal("delayed output diff set did not increase in size") } if cst.cs.db.getDelayedSiacoinOutputs(maturityHeight, id).Value.Cmp(dsco.Value) != 0 { t.Error("wrong delayed siacoin output value after committing a diff") } // Rewind the diff. cst.cs.commitDelayedSiacoinOutputDiff(dscod, modules.DiffRevert) if cst.cs.db.lenDelayedSiacoinOutputsHeight(maturityHeight) != initialDscosLen { t.Error("siacoin output diff set did not increase in size") } exists := cst.cs.db.inDelayedSiacoinOutputsHeight(maturityHeight, id) if exists { t.Error("siacoin output was not reverted") } // Restore the diff and then apply the inverse diff. cst.cs.commitDelayedSiacoinOutputDiff(dscod, modules.DiffApply) dscod.Direction = modules.DiffRevert cst.cs.commitDelayedSiacoinOutputDiff(dscod, modules.DiffApply) if cst.cs.db.lenDelayedSiacoinOutputsHeight(maturityHeight) != initialDscosLen { t.Error("siacoin output diff set did not increase in size") } exists = cst.cs.db.inDelayedSiacoinOutputsHeight(maturityHeight, id) if exists { t.Error("siacoin output was not reverted") } // Revert the inverse diff. cst.cs.commitDelayedSiacoinOutputDiff(dscod, modules.DiffRevert) if cst.cs.db.lenDelayedSiacoinOutputsHeight(maturityHeight) != initialDscosLen+1 { t.Error("siacoin output diff set did not increase in size") } if cst.cs.db.getDelayedSiacoinOutputs(maturityHeight, id).Value.Cmp(dsco.Value) != 0 { t.Error("wrong siacoin output value after committing a diff") } // Trigger an inconsistency check. defer func() { r := recover() if r != errBadCommitDelayedSiacoinOutputDiff { t.Error("expecting errBadCommitDelayedSiacoinOutputDiff, got", r) } }() // Try applying an apply diff that was already applied. (add an object // that already exists) dscod.Direction = modules.DiffApply // set the direction to apply cst.cs.commitDelayedSiacoinOutputDiff(dscod, modules.DiffApply) // apply an already existing delayed output. } */ /* // TestCommitSiafundPoolDiff probes the commitSiafundPoolDiff method of the // consensus set. func TestCommitSiafundPoolDiff(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestCommitSiafundPoolDiff") if err != nil { t.Fatal(err) } // Apply two siafund pool diffs, and then a diff with 0 change. Then revert // them all. initial := cst.cs.siafundPool adjusted1 := initial.Add(types.NewCurrency64(200)) adjusted2 := adjusted1.Add(types.NewCurrency64(500)) adjusted3 := adjusted2.Add(types.NewCurrency64(0)) sfpd1 := modules.SiafundPoolDiff{ Direction: modules.DiffApply, Previous: initial, Adjusted: adjusted1, } sfpd2 := modules.SiafundPoolDiff{ Direction: modules.DiffApply, Previous: adjusted1, Adjusted: adjusted2, } sfpd3 := modules.SiafundPoolDiff{ Direction: modules.DiffApply, Previous: adjusted2, Adjusted: adjusted3, } cst.cs.commitSiafundPoolDiff(sfpd1, modules.DiffApply) if cst.cs.siafundPool.Cmp(adjusted1) != 0 { t.Error("siafund pool was not adjusted correctly") } cst.cs.commitSiafundPoolDiff(sfpd2, modules.DiffApply) if cst.cs.siafundPool.Cmp(adjusted2) != 0 { t.Error("second siafund pool adjustment was flawed") } cst.cs.commitSiafundPoolDiff(sfpd3, modules.DiffApply) if cst.cs.siafundPool.Cmp(adjusted3) != 0 { t.Error("second siafund pool adjustment was flawed") } cst.cs.commitSiafundPoolDiff(sfpd3, modules.DiffRevert) if cst.cs.siafundPool.Cmp(adjusted2) != 0 { t.Error("reverting second adjustment was flawed") } cst.cs.commitSiafundPoolDiff(sfpd2, modules.DiffRevert) if cst.cs.siafundPool.Cmp(adjusted1) != 0 { t.Error("reverting second adjustment was flawed") } cst.cs.commitSiafundPoolDiff(sfpd1, modules.DiffRevert) if cst.cs.siafundPool.Cmp(initial) != 0 { t.Error("reverting first adjustment was flawed") } // Do a chaining set of panics. First apply a negative pool adjustment, // then revert the pool diffs in the wrong order, than apply the pool diffs // in the wrong order. defer func() { r := recover() if r != errApplySiafundPoolDiffMismatch { t.Error("expecting errApplySiafundPoolDiffMismatch, got", r) } }() defer func() { r := recover() if r != errRevertSiafundPoolDiffMismatch { t.Error("expecting errRevertSiafundPoolDiffMismatch, got", r) } cst.cs.commitSiafundPoolDiff(sfpd1, modules.DiffApply) }() defer func() { r := recover() if r != errNonApplySiafundPoolDiff { t.Error(r) } cst.cs.commitSiafundPoolDiff(sfpd1, modules.DiffRevert) }() defer func() { r := recover() if r != errNegativePoolAdjustment { t.Error("expecting errNegativePoolAdjustment, got", r) } sfpd2.Direction = modules.DiffRevert cst.cs.commitSiafundPoolDiff(sfpd2, modules.DiffApply) }() cst.cs.commitSiafundPoolDiff(sfpd1, modules.DiffApply) cst.cs.commitSiafundPoolDiff(sfpd2, modules.DiffApply) negativeAdjustment := adjusted2.Sub(types.NewCurrency64(100)) negativeSfpd := modules.SiafundPoolDiff{ Previous: adjusted3, Adjusted: negativeAdjustment, } cst.cs.commitSiafundPoolDiff(negativeSfpd, modules.DiffApply) } */ /* // TestDeleteObsoleteDelayedOutputMapsSanity probes the sanity checks of the // deleteObsoleteDelayedOutputMaps method of the consensus set. func TestDeleteObsoleteDelayedOutputMapsSanity(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestDeleteObsoleteDelayedOutputMapsSanity") if err != nil { t.Fatal(err) } pb := cst.cs.currentProcessedBlock() err = cst.cs.db.Update(func(tx *bolt.Tx) error { return commitDiffSet(tx, pb, modules.DiffRevert) }) if err != nil { t.Fatal(err) } defer func() { r := recover() if r == nil { t.Error("expecting an error after corrupting the database") } }() defer func() { r := recover() if r == nil { t.Error("expecting an error after corrupting the database") } // Trigger a panic by deleting a map with outputs in it during revert. err = cst.cs.db.Update(func(tx *bolt.Tx) error { return createUpcomingDelayedOutputMaps(tx, pb, modules.DiffApply) }) if err != nil { t.Fatal(err) } err = cst.cs.db.Update(func(tx *bolt.Tx) error { return commitNodeDiffs(tx, pb, modules.DiffApply) }) if err != nil { t.Fatal(err) } err = cst.cs.db.Update(func(tx *bolt.Tx) error { return deleteObsoleteDelayedOutputMaps(tx, pb, modules.DiffRevert) }) if err != nil { t.Fatal(err) } }() // Trigger a panic by deleting a map with outputs in it during apply. err = cst.cs.db.Update(func(tx *bolt.Tx) error { return deleteObsoleteDelayedOutputMaps(tx, pb, modules.DiffApply) }) if err != nil { t.Fatal(err) } } */ /* // TestGenerateAndApplyDiffSanity triggers the sanity checks in the // generateAndApplyDiff method of the consensus set. func TestGenerateAndApplyDiffSanity(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestGenerateAndApplyDiffSanity") if err != nil { t.Fatal(err) } pb := cst.cs.currentProcessedBlock() cst.cs.commitDiffSet(pb, modules.DiffRevert) defer func() { r := recover() if r != errRegenerateDiffs { t.Error("expected errRegenerateDiffs, got", r) } }() defer func() { r := recover() if r != errInvalidSuccessor { t.Error("expected errInvalidSuccessor, got", r) } // Trigger errRegenerteDiffs _ = cst.cs.generateAndApplyDiff(pb) }() // Trigger errInvalidSuccessor parent := cst.cs.db.getBlockMap(pb.Parent) parent.DiffsGenerated = false _ = cst.cs.generateAndApplyDiff(parent) } */ Sia-1.0.4/modules/consensus/fork.go000066400000000000000000000076271302033070500172150ustar00rootroot00000000000000package consensus import ( "errors" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/bolt" ) var ( errExternalRevert = errors.New("cannot revert to block outside of current path") ) // backtrackToCurrentPath traces backwards from 'pb' until it reaches a block // in the ConsensusSet's current path (the "common parent"). It returns the // (inclusive) set of blocks between the common parent and 'pb', starting from // the former. func backtrackToCurrentPath(tx *bolt.Tx, pb *processedBlock) []*processedBlock { path := []*processedBlock{pb} for { // Error is not checked in production code - an error can only indicate // that pb.Height > blockHeight(tx). currentPathID, err := getPath(tx, pb.Height) if currentPathID == pb.Block.ID() { break } // Sanity check - an error should only indicate that pb.Height > // blockHeight(tx). if build.DEBUG && err != nil && pb.Height <= blockHeight(tx) { panic(err) } // Prepend the next block to the list of blocks leading from the // current path to the input block. pb, err = getBlockMap(tx, pb.Block.ParentID) if build.DEBUG && err != nil { panic(err) } path = append([]*processedBlock{pb}, path...) } return path } // revertToBlock will revert blocks from the ConsensusSet's current path until // 'pb' is the current block. Blocks are returned in the order that they were // reverted. 'pb' is not reverted. func (cs *ConsensusSet) revertToBlock(tx *bolt.Tx, pb *processedBlock) (revertedBlocks []*processedBlock) { // Sanity check - make sure that pb is in the current path. currentPathID, err := getPath(tx, pb.Height) if build.DEBUG && (err != nil || currentPathID != pb.Block.ID()) { panic(errExternalRevert) } // Rewind blocks until 'pb' is the current block. for currentBlockID(tx) != pb.Block.ID() { block := currentProcessedBlock(tx) commitDiffSet(tx, block, modules.DiffRevert) revertedBlocks = append(revertedBlocks, block) // Sanity check - after removing a block, check that the consensus set // has maintained consistency. if build.Release == "testing" { cs.checkConsistency(tx) } else { cs.maybeCheckConsistency(tx) } } return revertedBlocks } // applyUntilBlock will successively apply the blocks between the consensus // set's current path and 'pb'. func (cs *ConsensusSet) applyUntilBlock(tx *bolt.Tx, pb *processedBlock) (appliedBlocks []*processedBlock, err error) { // Backtrack to the common parent of 'bn' and current path and then apply the new blocks. newPath := backtrackToCurrentPath(tx, pb) for _, block := range newPath[1:] { // If the diffs for this block have already been generated, apply diffs // directly instead of generating them. This is much faster. if block.DiffsGenerated { commitDiffSet(tx, block, modules.DiffApply) } else { err := generateAndApplyDiff(tx, block) if err != nil { // Mark the block as invalid. cs.dosBlocks[block.Block.ID()] = struct{}{} return nil, err } } appliedBlocks = append(appliedBlocks, block) // Sanity check - after applying a block, check that the consensus set // has maintained consistency. if build.Release == "testing" { cs.checkConsistency(tx) } else { cs.maybeCheckConsistency(tx) } } return appliedBlocks, nil } // forkBlockchain will move the consensus set onto the 'newBlock' fork. An // error will be returned if any of the blocks applied in the transition are // found to be invalid. forkBlockchain is atomic; the ConsensusSet is only // updated if the function returns nil. func (cs *ConsensusSet) forkBlockchain(tx *bolt.Tx, newBlock *processedBlock) (revertedBlocks, appliedBlocks []*processedBlock, err error) { commonParent := backtrackToCurrentPath(tx, newBlock)[0] revertedBlocks = cs.revertToBlock(tx, commonParent) appliedBlocks, err = cs.applyUntilBlock(tx, newBlock) if err != nil { return nil, nil, err } return revertedBlocks, appliedBlocks, nil } Sia-1.0.4/modules/consensus/fork_helpers_test.go000066400000000000000000000021161302033070500217620ustar00rootroot00000000000000package consensus import ( "github.com/NebulousLabs/bolt" ) // dbBacktrackToCurrentPath is a convenience function to call // backtrackToCurrentPath without a bolt.Tx. func (cs *ConsensusSet) dbBacktrackToCurrentPath(pb *processedBlock) (pbs []*processedBlock) { _ = cs.db.Update(func(tx *bolt.Tx) error { pbs = backtrackToCurrentPath(tx, pb) return nil }) return pbs } // dbRevertToNode is a convenience function to call revertToBlock without a // bolt.Tx. func (cs *ConsensusSet) dbRevertToNode(pb *processedBlock) (pbs []*processedBlock) { _ = cs.db.Update(func(tx *bolt.Tx) error { pbs = cs.revertToBlock(tx, pb) return nil }) return pbs } // dbForkBlockchain is a convenience function to call forkBlockchain without a // bolt.Tx. func (cs *ConsensusSet) dbForkBlockchain(pb *processedBlock) (revertedBlocks, appliedBlocks []*processedBlock, err error) { updateErr := cs.db.Update(func(tx *bolt.Tx) error { revertedBlocks, appliedBlocks, err = cs.forkBlockchain(tx, pb) return nil }) if updateErr != nil { panic(updateErr) } return revertedBlocks, appliedBlocks, err } Sia-1.0.4/modules/consensus/fork_test.go000066400000000000000000000052071302033070500202440ustar00rootroot00000000000000package consensus import ( "testing" "github.com/NebulousLabs/Sia/modules" ) // TestBacktrackToCurrentPath probes the backtrackToCurrentPath method of the // consensus set. func TestBacktrackToCurrentPath(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestBacktrackToCurrentPath") if err != nil { t.Fatal(err) } defer cst.Close() pb := cst.cs.dbCurrentProcessedBlock() // Backtrack from the current node to the blockchain. nodes := cst.cs.dbBacktrackToCurrentPath(pb) if len(nodes) != 1 { t.Fatal("backtracking to the current node gave incorrect result") } if nodes[0].Block.ID() != pb.Block.ID() { t.Error("backtrack returned the wrong node") } // Backtrack from a node that has diverted from the current blockchain. child0, _ := cst.miner.FindBlock() child1, _ := cst.miner.FindBlock() // Is the block not on hte current path. err = cst.cs.AcceptBlock(child0) if err != nil { t.Fatal(err) } err = cst.cs.AcceptBlock(child1) if err != modules.ErrNonExtendingBlock { t.Fatal(err) } pb, err = cst.cs.dbGetBlockMap(child1.ID()) if err != nil { t.Fatal(err) } nodes = cst.cs.dbBacktrackToCurrentPath(pb) if len(nodes) != 2 { t.Error("backtracking grabbed wrong number of nodes") } parent, err := cst.cs.dbGetBlockMap(pb.Block.ParentID) if err != nil { t.Fatal(err) } if nodes[0].Block.ID() != parent.Block.ID() { t.Error("grabbed the wrong block as the common block") } if nodes[1].Block.ID() != pb.Block.ID() { t.Error("backtracked from the wrong node") } } // TestRevertToNode probes the revertToBlock method of the consensus set. func TestRevertToNode(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestRevertToNode") if err != nil { t.Fatal(err) } defer cst.Close() pb := cst.cs.dbCurrentProcessedBlock() // Revert to a grandparent and verify the returned array is correct. parent, err := cst.cs.dbGetBlockMap(pb.Block.ParentID) if err != nil { t.Fatal(err) } grandParent, err := cst.cs.dbGetBlockMap(parent.Block.ParentID) if err != nil { t.Fatal(err) } revertedNodes := cst.cs.dbRevertToNode(grandParent) if len(revertedNodes) != 2 { t.Error("wrong number of nodes reverted") } if revertedNodes[0].Block.ID() != pb.Block.ID() { t.Error("wrong composition of reverted nodes") } if revertedNodes[1].Block.ID() != parent.Block.ID() { t.Error("wrong composition of reverted nodes") } // Trigger a panic by trying to revert to a node outside of the current // path. defer func() { r := recover() if r != errExternalRevert { t.Error(r) } }() cst.cs.dbRevertToNode(pb) } Sia-1.0.4/modules/consensus/maintenance.go000066400000000000000000000145311302033070500205260ustar00rootroot00000000000000package consensus import ( "errors" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) var ( errMissingFileContract = errors.New("storage proof submitted for non existing file contract") errOutputAlreadyMature = errors.New("delayed siacoin output is already in the matured outputs set") errPayoutsAlreadyPaid = errors.New("payouts are already in the consensus set") errStorageProofTiming = errors.New("missed proof triggered for file contract that is not expiring") ) // applyMinerPayouts adds a block's miner payouts to the consensus set as // delayed siacoin outputs. func applyMinerPayouts(tx *bolt.Tx, pb *processedBlock) { for i := range pb.Block.MinerPayouts { mpid := pb.Block.MinerPayoutID(uint64(i)) dscod := modules.DelayedSiacoinOutputDiff{ Direction: modules.DiffApply, ID: mpid, SiacoinOutput: pb.Block.MinerPayouts[i], MaturityHeight: pb.Height + types.MaturityDelay, } pb.DelayedSiacoinOutputDiffs = append(pb.DelayedSiacoinOutputDiffs, dscod) commitDelayedSiacoinOutputDiff(tx, dscod, modules.DiffApply) } } // applyMaturedSiacoinOutputs goes through the list of siacoin outputs that // have matured and adds them to the consensus set. This also updates the block // node diff set. func applyMaturedSiacoinOutputs(tx *bolt.Tx, pb *processedBlock) { // Skip this step if the blockchain is not old enough to have maturing // outputs. if pb.Height < types.MaturityDelay { return } // Iterate through the list of delayed siacoin outputs. Sometimes boltdb // has trouble if you delete elements in a bucket while iterating through // the bucket (and sometimes not - nondeterministic), so all of the // elements are collected into an array and then deleted after the bucket // scan is complete. bucketID := append(prefixDSCO, encoding.Marshal(pb.Height)...) var scods []modules.SiacoinOutputDiff var dscods []modules.DelayedSiacoinOutputDiff dbErr := tx.Bucket(bucketID).ForEach(func(idBytes, scoBytes []byte) error { // Decode the key-value pair into an id and a siacoin output. var id types.SiacoinOutputID var sco types.SiacoinOutput copy(id[:], idBytes) encErr := encoding.Unmarshal(scoBytes, &sco) if build.DEBUG && encErr != nil { panic(encErr) } // Sanity check - the output should not already be in siacoinOuptuts. if build.DEBUG && isSiacoinOutput(tx, id) { panic(errOutputAlreadyMature) } // Add the output to the ConsensusSet and record the diff in the // blockNode. scod := modules.SiacoinOutputDiff{ Direction: modules.DiffApply, ID: id, SiacoinOutput: sco, } scods = append(scods, scod) // Create the dscod and add it to the list of dscods that should be // deleted. dscod := modules.DelayedSiacoinOutputDiff{ Direction: modules.DiffRevert, ID: id, SiacoinOutput: sco, MaturityHeight: pb.Height, } dscods = append(dscods, dscod) return nil }) if build.DEBUG && dbErr != nil { panic(dbErr) } for _, scod := range scods { pb.SiacoinOutputDiffs = append(pb.SiacoinOutputDiffs, scod) commitSiacoinOutputDiff(tx, scod, modules.DiffApply) } for _, dscod := range dscods { pb.DelayedSiacoinOutputDiffs = append(pb.DelayedSiacoinOutputDiffs, dscod) commitDelayedSiacoinOutputDiff(tx, dscod, modules.DiffApply) } deleteDSCOBucket(tx, pb.Height) } // applyMissedStorageProof adds the outputs and diffs that result from a file // contract expiring. func applyMissedStorageProof(tx *bolt.Tx, pb *processedBlock, fcid types.FileContractID) (dscods []modules.DelayedSiacoinOutputDiff, fcd modules.FileContractDiff) { // Sanity checks. fc, err := getFileContract(tx, fcid) if build.DEBUG && err != nil { panic(err) } if build.DEBUG { // Check that the file contract in question expires at pb.Height. if fc.WindowEnd != pb.Height { panic(errStorageProofTiming) } } // Add all of the outputs in the missed proof outputs to the consensus set. for i, mpo := range fc.MissedProofOutputs { // Sanity check - output should not already exist. spoid := fcid.StorageProofOutputID(types.ProofMissed, uint64(i)) if build.DEBUG && isSiacoinOutput(tx, spoid) { panic(errPayoutsAlreadyPaid) } // Don't add the output if the value is zero. dscod := modules.DelayedSiacoinOutputDiff{ Direction: modules.DiffApply, ID: spoid, SiacoinOutput: mpo, MaturityHeight: pb.Height + types.MaturityDelay, } dscods = append(dscods, dscod) } // Remove the file contract from the consensus set and record the diff in // the blockNode. fcd = modules.FileContractDiff{ Direction: modules.DiffRevert, ID: fcid, FileContract: fc, } return dscods, fcd } // applyFileContractMaintenance looks for all of the file contracts that have // expired without an appropriate storage proof, and calls 'applyMissedProof' // for the file contract. func applyFileContractMaintenance(tx *bolt.Tx, pb *processedBlock) { // Get the bucket pointing to all of the expiring file contracts. fceBucketID := append(prefixFCEX, encoding.Marshal(pb.Height)...) fceBucket := tx.Bucket(fceBucketID) // Finish if there are no expiring file contracts. if fceBucket == nil { return } var dscods []modules.DelayedSiacoinOutputDiff var fcds []modules.FileContractDiff err := fceBucket.ForEach(func(keyBytes, valBytes []byte) error { var id types.FileContractID copy(id[:], keyBytes) amspDSCODS, fcd := applyMissedStorageProof(tx, pb, id) fcds = append(fcds, fcd) dscods = append(dscods, amspDSCODS...) return nil }) if build.DEBUG && err != nil { panic(err) } for _, dscod := range dscods { pb.DelayedSiacoinOutputDiffs = append(pb.DelayedSiacoinOutputDiffs, dscod) commitDelayedSiacoinOutputDiff(tx, dscod, modules.DiffApply) } for _, fcd := range fcds { pb.FileContractDiffs = append(pb.FileContractDiffs, fcd) commitFileContractDiff(tx, fcd, modules.DiffApply) } err = tx.DeleteBucket(fceBucketID) if build.DEBUG && err != nil { panic(err) } } // applyMaintenance applies block-level alterations to the consensus set. // Maintenance is applied after all of the transcations for the block have been // applied. func applyMaintenance(tx *bolt.Tx, pb *processedBlock) { applyMinerPayouts(tx, pb) applyMaturedSiacoinOutputs(tx, pb) applyFileContractMaintenance(tx, pb) } Sia-1.0.4/modules/consensus/maintenance_test.go000066400000000000000000000214421302033070500215640ustar00rootroot00000000000000package consensus /* import ( "testing" "github.com/NebulousLabs/bolt" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // TestApplyMinerPayouts probes the applyMinerPayouts method of the consensus // set. func TestApplyMinerPayouts(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestApplyMinerPayouts") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node with a single miner payout. pb := new(processedBlock) pb.Height = cst.cs.dbBlockHeight() pb.Block.Timestamp = 2 // MinerPayout id is determined by block id + index; add uniqueness to the block id. pb.Block.MinerPayouts = append(pb.Block.MinerPayouts, types.SiacoinOutput{Value: types.NewCurrency64(12)}) mpid0 := pb.Block.MinerPayoutID(0) // Apply the single miner payout. _ = cst.cs.db.Update(func(tx *bolt.Tx) error { applyMinerPayouts(tx, pb) return nil }) exists := cst.cs.db.inDelayedSiacoinOutputsHeight(cst.cs.dbBlockHeight()+types.MaturityDelay, mpid0) if !exists { t.Error("miner payout was not created in the delayed outputs set") } dsco, err := cst.cs.dbGetDSCO(cst.cs.dbBlockHeight()+types.MaturityDelay, mpid0) if err != nil { t.Fatal(err) } if dsco.Value.Cmp64(12) != 0 { t.Error("miner payout created with wrong currency value") } exists = cst.cs.db.inSiacoinOutputs(mpid0) if exists { t.Error("miner payout was added to the siacoin output set") } if cst.cs.db.lenDelayedSiacoinOutputsHeight(cst.cs.dbBlockHeight()+types.MaturityDelay) != 2 { // 1 for consensus set creation, 1 for the output that just got added. t.Error("wrong number of delayed siacoin outputs in consensus set") } if len(pb.DelayedSiacoinOutputDiffs) != 1 { t.Fatal("block node did not get the delayed siacoin output diff") } if pb.DelayedSiacoinOutputDiffs[0].Direction != modules.DiffApply { t.Error("delayed siacoin output diff has the wrong direction") } if pb.DelayedSiacoinOutputDiffs[0].ID != mpid0 { t.Error("delayed siacoin output diff has wrong id") } // Apply a processed block with two miner payouts. pb2 := new(processedBlock) pb2.Height = cst.cs.dbBlockHeight() pb2.Block.Timestamp = 5 // MinerPayout id is determined by block id + index; add uniqueness to the block id. pb2.Block.MinerPayouts = []types.SiacoinOutput{ {Value: types.NewCurrency64(5)}, {Value: types.NewCurrency64(10)}, } mpid1 := pb2.Block.MinerPayoutID(0) mpid2 := pb2.Block.MinerPayoutID(1) _ = cst.cs.db.Update(func(tx *bolt.Tx) error { applyMinerPayouts(tx, pb2) return nil }) exists = cst.cs.db.inDelayedSiacoinOutputsHeight(cst.cs.dbBlockHeight()+types.MaturityDelay, mpid1) if !exists { t.Error("delayed siacoin output was not created") } exists = cst.cs.db.inDelayedSiacoinOutputsHeight(cst.cs.dbBlockHeight()+types.MaturityDelay, mpid2) if !exists { t.Error("delayed siacoin output was not created") } if len(pb2.DelayedSiacoinOutputDiffs) != 2 { t.Error("block node should have 2 delayed outputs") } // Trigger a panic where the miner payouts have already been applied. defer func() { r := recover() if r == nil { t.Error("expecting error after corrupting database") } }() defer func() { r := recover() if r == nil { t.Error("expecting error after corrupting database") } cst.cs.db.rmDelayedSiacoinOutputsHeight(pb.Height+types.MaturityDelay, mpid0) cst.cs.db.addSiacoinOutputs(mpid0, types.SiacoinOutput{}) _ = cst.cs.db.Update(func(tx *bolt.Tx) error { applyMinerPayouts(tx, pb) return nil }) }() _ = cst.cs.db.Update(func(tx *bolt.Tx) error { applyMinerPayouts(tx, pb) return nil }) } // TestApplyMaturedSiacoinOutputs probes the applyMaturedSiacoinOutputs method // of the consensus set. func TestApplyMaturedSiacoinOutputs(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestApplyMaturedSiacoinOutputs") if err != nil { t.Fatal(err) } defer cst.closeCst() pb := cst.cs.dbCurrentProcessedBlock() // Trigger the sanity check concerning already-matured outputs. defer func() { r := recover() if r != errOutputAlreadyMature { t.Error(r) } }() cst.cs.db.addSiacoinOutputs(types.SiacoinOutputID{}, types.SiacoinOutput{}) _ = cst.cs.db.Update(func(tx *bolt.Tx) error { createDSCOBucket(tx, pb.Height) return nil }) cst.cs.db.addDelayedSiacoinOutputsHeight(pb.Height, types.SiacoinOutputID{}, types.SiacoinOutput{}) _ = cst.cs.db.Update(func(tx *bolt.Tx) error { applyMaturedSiacoinOutputs(tx, pb) return nil }) } // TestApplyMissedStorageProof probes the applyMissedStorageProof method of the // consensus set. func TestApplyMissedStorageProof(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestApplyMissedStorageProof") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node. pb := new(processedBlock) pb.Height = cst.cs.height() // Create a file contract that's expiring and has 1 missed proof output. expiringFC := types.FileContract{ Payout: types.NewCurrency64(300e3), WindowEnd: pb.Height, MissedProofOutputs: []types.SiacoinOutput{{Value: types.NewCurrency64(290e3)}}, } // Assign the contract a 0-id. cst.cs.db.addFileContracts(types.FileContractID{}, expiringFC) cst.cs.db.addFCExpirations(pb.Height) cst.cs.db.addFCExpirationsHeight(pb.Height, types.FileContractID{}) cst.cs.applyMissedStorageProof(pb, types.FileContractID{}) exists := cst.cs.db.inFileContracts(types.FileContractID{}) if exists { t.Error("file contract was not consumed in missed storage proof") } spoid := types.FileContractID{}.StorageProofOutputID(types.ProofMissed, 0) exists = cst.cs.db.inDelayedSiacoinOutputsHeight(pb.Height+types.MaturityDelay, spoid) if !exists { t.Error("missed proof output was never created") } exists = cst.cs.db.inSiacoinOutputs(spoid) if exists { t.Error("storage proof output made it into the siacoin output set") } exists = cst.cs.db.inFileContracts(types.FileContractID{}) if exists { t.Error("file contract remains after expiration") } // Trigger the debug panics. // not exist. defer func() { r := recover() if r != errNilItem { t.Error(r) } }() defer func() { r := recover() if r != errNilItem { t.Error(r) } // Trigger errMissingFileContract cst.cs.applyMissedStorageProof(pb, types.FileContractID(spoid)) }() defer func() { r := recover() if r != errNilItem { t.Error(r) } // Trigger errStorageProofTiming expiringFC.WindowEnd = 0 cst.cs.applyMissedStorageProof(pb, types.FileContractID{}) }() defer func() { r := recover() if r != errNilItem { t.Error(r) } // Trigger errPayoutsAlreadyPaid from siacoin outputs. cst.cs.db.rmDelayedSiacoinOutputsHeight(pb.Height+types.MaturityDelay, spoid) cst.cs.db.addSiacoinOutputs(spoid, types.SiacoinOutput{}) cst.cs.applyMissedStorageProof(pb, types.FileContractID{}) }() // Trigger errPayoutsAlreadyPaid from delayed outputs. cst.cs.db.rmFileContracts(types.FileContractID{}) cst.cs.db.addFileContracts(types.FileContractID{}, expiringFC) cst.cs.db.addDelayedSiacoinOutputsHeight(pb.Height+types.MaturityDelay, spoid, types.SiacoinOutput{}) cst.cs.applyMissedStorageProof(pb, types.FileContractID{}) } */ // TestApplyFileContractMaintenance probes the applyFileContractMaintenance // method of the consensus set. /* func TestApplyFileContractMaintenance(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestApplyMissedStorageProof") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a block node. pb := new(processedBlock) pb.Height = cst.cs.height() // Create a file contract that's expiring and has 1 missed proof output. expiringFC := types.FileContract{ Payout: types.NewCurrency64(300e3), WindowEnd: pb.Height, MissedProofOutputs: []types.SiacoinOutput{{Value: types.NewCurrency64(290e3)}}, } // Assign the contract a 0-id. cst.cs.db.addFileContracts(types.FileContractID{}, expiringFC) cst.cs.db.addFCExpirations(pb.Height) cst.cs.db.addFCExpirationsHeight(pb.Height, types.FileContractID{}) err = cst.cs.db.Update(func(tx *bolt.Tx) error { applyFileContractMaintenance(tx, pb) return nil }) if err != nil { t.Fatal(err) } exists := cst.cs.db.inFileContracts(types.FileContractID{}) if exists { t.Error("file contract was not consumed in missed storage proof") } spoid := types.FileContractID{}.StorageProofOutputID(types.ProofMissed, 0) exists = cst.cs.db.inDelayedSiacoinOutputsHeight(pb.Height+types.MaturityDelay, spoid) if !exists { t.Error("missed proof output was never created") } exists = cst.cs.db.inSiacoinOutputs(spoid) if exists { t.Error("storage proof output made it into the siacoin output set") } exists = cst.cs.db.inFileContracts(types.FileContractID{}) if exists { t.Error("file contract remains after expiration") } } */ Sia-1.0.4/modules/consensus/persist.go000066400000000000000000000047461302033070500177440ustar00rootroot00000000000000package consensus import ( "errors" "fmt" "os" "path/filepath" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/bolt" ) const ( // DatabaseFilename contains the filename of the database that will be used // when managing consensus. DatabaseFilename = modules.ConsensusDir + ".db" logFile = modules.ConsensusDir + ".log" ) // loadDB pulls all the blocks that have been saved to disk into memory, using // them to fill out the ConsensusSet. func (cs *ConsensusSet) loadDB() error { // Open the database - a new bolt database will be created if none exists. err := cs.openDB(filepath.Join(cs.persistDir, DatabaseFilename)) if err != nil { return err } // Walk through initialization for Sia. return cs.db.Update(func(tx *bolt.Tx) error { // Check if the database has been initialized. if !dbInitialized(tx) { return cs.initDB(tx) } // Check that inconsistencies have not been detected in the database. if inconsistencyDetected(tx) { return errors.New("database contains inconsistencies") } // Check that the genesis block is correct - typically only incorrect // in the event of developer binaries vs. release binaires. genesisID, err := getPath(tx, 0) if build.DEBUG && err != nil { panic(err) } if genesisID != cs.blockRoot.Block.ID() { return errors.New("Blockchain has wrong genesis block, exiting.") } return nil }) } // initPersist initializes the persistence structures of the consensus set, in // particular loading the database and preparing to manage subscribers. func (cs *ConsensusSet) initPersist() error { // Create the consensus directory. err := os.MkdirAll(cs.persistDir, 0700) if err != nil { return err } // Initialize the logger. cs.log, err = persist.NewFileLogger(filepath.Join(cs.persistDir, logFile)) if err != nil { return err } // Set up closing the logger. cs.tg.AfterStop(func() { err := cs.log.Close() if err != nil { // State of the logger is unknown, a println will suffice. fmt.Println("Error shutting down consensus set logger:", err) } }) // Try to load an existing database from disk - a new one will be created // if one does not exist. err = cs.loadDB() if err != nil { return err } // Set up the closing of the database. cs.tg.AfterStop(func() { err := cs.db.Close() if err != nil { cs.log.Println("ERROR: Unable to close consensus set database at shutdown:", err) } }) return nil } Sia-1.0.4/modules/consensus/persist_test.go000066400000000000000000000021351302033070500207710ustar00rootroot00000000000000package consensus import ( "path/filepath" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/gateway" ) // TestSaveLoad populates a blockchain, saves it, loads it, and checks // the consensus set hash before and after func TestSaveLoad(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestSaveLoad") if err != nil { t.Fatal(err) } defer cst.Close() cst.testBlockSuite() oldHash := cst.cs.dbConsensusChecksum() cst.cs.Close() // Reassigning this will lose subscribers and such, but we // just want to call load and get a hash g, err := gateway.New("localhost:0", false, build.TempDir(modules.ConsensusDir, "TestSaveLoad", modules.GatewayDir)) if err != nil { t.Fatal(err) } d := filepath.Join(build.SiaTestingDir, modules.ConsensusDir, "TestSaveLoad", modules.ConsensusDir) cst.cs, err = New(g, false, d) if err != nil { t.Fatal(err) } newHash := cst.cs.dbConsensusChecksum() if oldHash != newHash { t.Fatal("consensus set hash changed after load") } } Sia-1.0.4/modules/consensus/processedblock.go000066400000000000000000000125351302033070500212500ustar00rootroot00000000000000package consensus import ( "math/big" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // SurpassThreshold is a percentage that dictates how much heavier a competing // chain has to be before the node will switch to mining on that chain. This is // not a consensus rule. This percentage is only applied to the most recent // block, not the entire chain; see blockNode.heavierThan. // // If no threshold were in place, it would be possible to manipulate a block's // timestamp to produce a sufficiently heavier block. var SurpassThreshold = big.NewRat(20, 100) // processedBlock is a copy/rename of blockNode, with the pointers to // other blockNodes replaced with block ID's, and all the fields // exported, so that a block node can be marshalled type processedBlock struct { Block types.Block Height types.BlockHeight Depth types.Target ChildTarget types.Target DiffsGenerated bool SiacoinOutputDiffs []modules.SiacoinOutputDiff FileContractDiffs []modules.FileContractDiff SiafundOutputDiffs []modules.SiafundOutputDiff DelayedSiacoinOutputDiffs []modules.DelayedSiacoinOutputDiff SiafundPoolDiffs []modules.SiafundPoolDiff ConsensusChecksum crypto.Hash } // heavierThan returns true if the blockNode is sufficiently heavier than // 'cmp'. 'cmp' is expected to be the current block node. "Sufficient" means // that the weight of 'bn' exceeds the weight of 'cmp' by: // (the target of 'cmp' * 'Surpass Threshold') func (pb *processedBlock) heavierThan(cmp *processedBlock) bool { requirement := cmp.Depth.AddDifficulties(cmp.ChildTarget.MulDifficulty(SurpassThreshold)) return requirement.Cmp(pb.Depth) > 0 // Inversed, because the smaller target is actually heavier. } // childDepth returns the depth of a blockNode's child nodes. The depth is the // "sum" of the current depth and current difficulty. See target.Add for more // detailed information. func (pb *processedBlock) childDepth() types.Target { return pb.Depth.AddDifficulties(pb.ChildTarget) } // targetAdjustmentBase returns the magnitude that the target should be // adjusted by before a clamp is applied. func (cs *ConsensusSet) targetAdjustmentBase(blockMap *bolt.Bucket, pb *processedBlock) *big.Rat { // Grab the block that was generated 'TargetWindow' blocks prior to the // parent. If there are not 'TargetWindow' blocks yet, stop at the genesis // block. var windowSize types.BlockHeight parent := pb.Block.ParentID current := pb.Block.ID() for windowSize = 0; windowSize < types.TargetWindow && parent != (types.BlockID{}); windowSize++ { current = parent copy(parent[:], blockMap.Get(parent[:])[:32]) } timestamp := types.Timestamp(encoding.DecUint64(blockMap.Get(current[:])[40:48])) // The target of a child is determined by the amount of time that has // passed between the generation of its immediate parent and its // TargetWindow'th parent. The expected amount of seconds to have passed is // TargetWindow*BlockFrequency. The target is adjusted in proportion to how // time has passed vs. the expected amount of time to have passed. // // The target is converted to a big.Rat to provide infinite precision // during the calculation. The big.Rat is just the int representation of a // target. timePassed := pb.Block.Timestamp - timestamp expectedTimePassed := types.BlockFrequency * windowSize return big.NewRat(int64(timePassed), int64(expectedTimePassed)) } // clampTargetAdjustment returns a clamped version of the base adjustment // value. The clamp keeps the maximum adjustment to ~7x every 2000 blocks. This // ensures that raising and lowering the difficulty requires a minimum amount // of total work, which prevents certain classes of difficulty adjusting // attacks. func clampTargetAdjustment(base *big.Rat) *big.Rat { if base.Cmp(types.MaxAdjustmentUp) > 0 { return types.MaxAdjustmentUp } else if base.Cmp(types.MaxAdjustmentDown) < 0 { return types.MaxAdjustmentDown } return base } // setChildTarget computes the target of a blockNode's child. All children of a node // have the same target. func (cs *ConsensusSet) setChildTarget(blockMap *bolt.Bucket, pb *processedBlock) { // Fetch the parent block. var parent processedBlock parentBytes := blockMap.Get(pb.Block.ParentID[:]) err := encoding.Unmarshal(parentBytes, &parent) if build.DEBUG && err != nil { panic(err) } if pb.Height%(types.TargetWindow/2) != 0 { pb.ChildTarget = parent.ChildTarget return } adjustment := clampTargetAdjustment(cs.targetAdjustmentBase(blockMap, pb)) adjustedRatTarget := new(big.Rat).Mul(parent.ChildTarget.Rat(), adjustment) pb.ChildTarget = types.RatToTarget(adjustedRatTarget) } // newChild creates a blockNode from a block and adds it to the parent's set of // children. The new node is also returned. It necessairly modifies the database func (cs *ConsensusSet) newChild(tx *bolt.Tx, pb *processedBlock, b types.Block) *processedBlock { // Create the child node. childID := b.ID() child := &processedBlock{ Block: b, Height: pb.Height + 1, Depth: pb.childDepth(), } blockMap := tx.Bucket(BlockMap) cs.setChildTarget(blockMap, child) err := blockMap.Put(childID[:], encoding.Marshal(*child)) if build.DEBUG && err != nil { panic(err) } return child } Sia-1.0.4/modules/consensus/processedblock_test.go000066400000000000000000000260121302033070500223020ustar00rootroot00000000000000package consensus import ( "path/filepath" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/modules/miner" "github.com/NebulousLabs/Sia/modules/transactionpool" "github.com/NebulousLabs/Sia/modules/wallet" "github.com/NebulousLabs/Sia/types" ) // TestIntegrationMinimumValidChildTimestamp probes the // MinimumValidChildTimestamp method of the consensus type. func TestIntegrationMinimumValidChildTimestamp(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // Create a custom consensus set to control the blocks. testdir := build.TempDir(modules.ConsensusDir, "TestIntegrationMinimumValidChildTimestamp") g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { t.Fatal(err) } cs, err := New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { t.Fatal(err) } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { t.Fatal(err) } w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { t.Fatal(err) } key, err := crypto.GenerateTwofishKey() if err != nil { t.Fatal(err) } _, err = w.Encrypt(key) if err != nil { t.Fatal(err) } err = w.Unlock(key) if err != nil { t.Fatal(err) } m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.MinerDir)) if err != nil { t.Fatal(err) } defer g.Close() // The earliest child timestamp of the genesis block should be the // timestamp of the genesis block. genesisTime := cs.blockRoot.Block.Timestamp earliest, ok := cs.MinimumValidChildTimestamp(cs.blockRoot.Block.ID()) if !ok || genesisTime != earliest { t.Error("genesis block earliest timestamp producing unexpected results") } timestampOffsets := []types.Timestamp{1, 3, 2, 5, 4, 6, 7, 8, 9, 10} blockIDs := []types.BlockID{cs.blockRoot.Block.ID()} for _, offset := range timestampOffsets { bfw, target, err := m.BlockForWork() if err != nil { t.Fatal(err) } bfw.Timestamp = genesisTime + offset solvedBlock, _ := m.SolveBlock(bfw, target) err = cs.AcceptBlock(solvedBlock) if err != nil { t.Fatal(err) } blockIDs = append(blockIDs, solvedBlock.ID()) } // Median should be genesisTime for 6th block. earliest, ok = cs.MinimumValidChildTimestamp(blockIDs[5]) if !ok || earliest != genesisTime { t.Error("incorrect child timestamp") } // Median should be genesisTime+1 for 7th block. earliest, ok = cs.MinimumValidChildTimestamp(blockIDs[6]) if !ok || earliest != genesisTime+1 { t.Error("incorrect child timestamp") } // Median should be genesisTime + 5 for pb11. earliest, ok = cs.MinimumValidChildTimestamp(blockIDs[10]) if !ok || earliest != genesisTime+5 { t.Error("incorrect child timestamp") } } // TestUnitHeavierThan probes the heavierThan method of the processedBlock type. func TestUnitHeavierThan(t *testing.T) { // Create a light node. pbLight := new(processedBlock) pbLight.Depth[0] = 64 pbLight.ChildTarget[0] = 200 // Create a node that's heavier, but not enough to beat the surpass // threshold. pbMiddle := new(processedBlock) pbMiddle.Depth[0] = 60 pbMiddle.ChildTarget[0] = 200 // Create a node that's heavy enough to break the surpass threshold. pbHeavy := new(processedBlock) pbHeavy.Depth[0] = 16 pbHeavy.ChildTarget[0] = 200 // pbLight should not be heavier than pbHeavy. if pbLight.heavierThan(pbHeavy) { t.Error("light heavier than heavy") } // pbLight should not be heavier than middle. if pbLight.heavierThan(pbMiddle) { t.Error("light heavier than middle") } // pbLight should not be heavier than itself. if pbLight.heavierThan(pbLight) { t.Error("light heavier than itself") } // pbMiddle should not be heavier than pbLight. if pbMiddle.heavierThan(pbLight) { t.Error("middle heaver than light - surpass threshold should not have been broken") } // pbHeavy should be heaver than pbLight. if !pbHeavy.heavierThan(pbLight) { t.Error("heavy is not heavier than light") } // pbHeavy should be heavier than pbMiddle. if !pbHeavy.heavierThan(pbMiddle) { t.Error("heavy is not heavier than middle") } } // TestChildDepth probes the childDeath method of the blockNode type. func TestChildDepth(t *testing.T) { // Try adding to equal weight nodes, result should be half. pb := new(processedBlock) pb.Depth[0] = 64 pb.ChildTarget[0] = 64 childDepth := pb.childDepth() if childDepth[0] != 32 { t.Error("unexpected child depth") } // Try adding nodes of different weights. pb.Depth[0] = 24 pb.ChildTarget[0] = 48 childDepth = pb.childDepth() if childDepth[0] != 16 { t.Error("unexpected child depth") } } /* // TestTargetAdjustmentBase probes the targetAdjustmentBase method of the block // node type. func TestTargetAdjustmentBase(t *testing.T) { cst, err := createConsensusSetTester("TestTargetAdjustmentBase") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a genesis node at timestamp 10,000 genesisNode := &processedBlock{ Block: types.Block{Timestamp: 10000}, } cst.cs.db.addBlockMap(genesisNode) exactTimeNode := &processedBlock{ Block: types.Block{ Nonce: types.BlockNonce{1, 0, 0, 0, 0, 0, 0, 0}, Timestamp: types.Timestamp(10000 + types.BlockFrequency), }, } exactTimeNode.Parent = genesisNode.Block.ID() cst.cs.db.addBlockMap(exactTimeNode) // Base adjustment for the exactTimeNode should be 1. adjustment, exact := cst.cs.targetAdjustmentBase(exactTimeNode).Float64() if !exact { t.Fatal("did not get an exact target adjustment") } if adjustment != 1 { t.Error("block did not adjust itself to the same target") } // Create a double-speed node and get the base adjustment. doubleSpeedNode := &processedBlock{ Block: types.Block{Timestamp: types.Timestamp(10000 + types.BlockFrequency)}, } doubleSpeedNode.Parent = exactTimeNode.Block.ID() cst.cs.db.addBlockMap(doubleSpeedNode) adjustment, exact = cst.cs.targetAdjustmentBase(doubleSpeedNode).Float64() if !exact { t.Fatal("did not get an exact adjustment") } if adjustment != 0.5 { t.Error("double speed node did not get a base to halve the target") } // Create a half-speed node and get the base adjustment. halfSpeedNode := &processedBlock{ Block: types.Block{Timestamp: types.Timestamp(10000 + types.BlockFrequency*6)}, } halfSpeedNode.Parent = doubleSpeedNode.Block.ID() cst.cs.db.addBlockMap(halfSpeedNode) adjustment, exact = cst.cs.targetAdjustmentBase(halfSpeedNode).Float64() if !exact { t.Fatal("did not get an exact adjustment") } if adjustment != 2 { t.Error("double speed node did not get a base to halve the target") } if testing.Short() { t.SkipNow() } // Create a chain of nodes so that the genesis node is no longer the point // of comparison. comparisonNode := &processedBlock{ Block: types.Block{Timestamp: 125000}, } comparisonNode.Parent = halfSpeedNode.Block.ID() cst.cs.db.addBlockMap(comparisonNode) startingNode := comparisonNode for i := types.BlockHeight(0); i < types.TargetWindow; i++ { newNode := new(processedBlock) newNode.Parent = startingNode.Block.ID() newNode.Block.Nonce = types.BlockNonce{byte(i), byte(i / 256), 0, 0, 0, 0, 0, 0} cst.cs.db.addBlockMap(newNode) startingNode = newNode } startingNode.Block.Timestamp = types.Timestamp(125000 + types.BlockFrequency*types.TargetWindow) adjustment, exact = cst.cs.targetAdjustmentBase(startingNode).Float64() if !exact { t.Error("failed to get exact result") } if adjustment != 1 { t.Error("got wrong long-range adjustment") } startingNode.Block.Timestamp = types.Timestamp(125000 + 2*types.BlockFrequency*types.TargetWindow) adjustment, exact = cst.cs.targetAdjustmentBase(startingNode).Float64() if !exact { t.Error("failed to get exact result") } if adjustment != 2 { t.Error("got wrong long-range adjustment") } } // TestClampTargetAdjustment probes the clampTargetAdjustment function. func TestClampTargetAdjustment(t *testing.T) { // Check that the MaxAdjustmentUp and MaxAdjustmentDown constants match the // test's expectations. if types.MaxAdjustmentUp.Cmp(big.NewRat(10001, 10000)) != 0 { t.Fatal("MaxAdjustmentUp changed - test now invalid") } if types.MaxAdjustmentDown.Cmp(big.NewRat(9999, 10000)) != 0 { t.Fatal("MaxAdjustmentDown changed - test now invalid") } // Check high and low clamping. initial := big.NewRat(2, 1) clamped := clampTargetAdjustment(initial) if clamped.Cmp(big.NewRat(10001, 10000)) != 0 { t.Error("clamp not applied to large target adjustment") } initial = big.NewRat(1, 2) clamped = clampTargetAdjustment(initial) if clamped.Cmp(big.NewRat(9999, 10000)) != 0 { t.Error("clamp not applied to small target adjustment") } // Check middle clamping (or lack thereof). initial = big.NewRat(10002, 10001) clamped = clampTargetAdjustment(initial) if clamped.Cmp(initial) != 0 { t.Error("clamp applied to safe target adjustment") } initial = big.NewRat(99999, 100000) clamped = clampTargetAdjustment(initial) if clamped.Cmp(initial) != 0 { t.Error("clamp applied to safe target adjustment") } } // TestSetChildTarget probes the setChildTarget method of the block node type. func TestSetChildTarget(t *testing.T) { cst, err := createConsensusSetTester("TestSetChildTarget") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a genesis node and a child that took 2x as long as expected. genesisNode := &processedBlock{ Block: types.Block{Timestamp: 10000}, } genesisNode.ChildTarget[0] = 64 cst.cs.db.addBlockMap(genesisNode) doubleTimeNode := &processedBlock{ Block: types.Block{Timestamp: types.Timestamp(10000 + types.BlockFrequency*2)}, } doubleTimeNode.Parent = genesisNode.Block.ID() cst.cs.db.addBlockMap(doubleTimeNode) // Check the resulting childTarget of the new node and see that the clamp // was applied. cst.cs.setChildTarget(doubleTimeNode) if doubleTimeNode.ChildTarget.Cmp(genesisNode.ChildTarget) <= 0 { t.Error("double time node target did not increase") } fullAdjustment := genesisNode.ChildTarget.MulDifficulty(big.NewRat(1, 2)) if doubleTimeNode.ChildTarget.Cmp(fullAdjustment) >= 0 { t.Error("clamp was not applied when adjusting target") } } // TestNewChild probes the newChild method of the block node type. func TestNewChild(t *testing.T) { cst, err := createConsensusSetTester("TestNewChild") if err != nil { t.Fatal(err) } defer cst.closeCst() parent := &processedBlock{ Height: 12, } parent.Depth[0] = 45 parent.Block.Timestamp = 100 parent.ChildTarget[0] = 90 cst.cs.db.addBlockMap(parent) child := cst.cs.newChild(parent, types.Block{Timestamp: types.Timestamp(100 + types.BlockFrequency)}) if child.Parent != parent.Block.ID() { t.Error("parent-child relationship incorrect") } if child.Height != 13 { t.Error("child height set incorrectly") } var expectedDepth types.Target expectedDepth[0] = 30 if child.Depth.Cmp(expectedDepth) != 0 { t.Error("child depth did not adjust correctly") } if child.ChildTarget.Cmp(parent.ChildTarget) != 0 { t.Error("child childTarget not adjusted correctly") } } */ Sia-1.0.4/modules/consensus/subscribe.go000066400000000000000000000174001302033070500202230ustar00rootroot00000000000000package consensus import ( "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/bolt" ) // computeConsensusChange computes the consensus change from the change entry // at index 'i' in the change log. If i is out of bounds, an error is returned. func (cs *ConsensusSet) computeConsensusChange(tx *bolt.Tx, ce changeEntry) (modules.ConsensusChange, error) { cc := modules.ConsensusChange{ ID: ce.ID(), } for _, revertedBlockID := range ce.RevertedBlocks { revertedBlock, err := getBlockMap(tx, revertedBlockID) if err != nil { cs.log.Critical("getBlockMap failed in computeConsensusChange:", err) return modules.ConsensusChange{}, err } // Because the direction is 'revert', the order of the diffs needs to // be flipped and the direction of the diffs also needs to be flipped. cc.RevertedBlocks = append(cc.RevertedBlocks, revertedBlock.Block) for i := len(revertedBlock.SiacoinOutputDiffs) - 1; i >= 0; i-- { scod := revertedBlock.SiacoinOutputDiffs[i] scod.Direction = !scod.Direction cc.SiacoinOutputDiffs = append(cc.SiacoinOutputDiffs, scod) } for i := len(revertedBlock.FileContractDiffs) - 1; i >= 0; i-- { fcd := revertedBlock.FileContractDiffs[i] fcd.Direction = !fcd.Direction cc.FileContractDiffs = append(cc.FileContractDiffs, fcd) } for i := len(revertedBlock.SiafundOutputDiffs) - 1; i >= 0; i-- { sfod := revertedBlock.SiafundOutputDiffs[i] sfod.Direction = !sfod.Direction cc.SiafundOutputDiffs = append(cc.SiafundOutputDiffs, sfod) } for i := len(revertedBlock.DelayedSiacoinOutputDiffs) - 1; i >= 0; i-- { dscod := revertedBlock.DelayedSiacoinOutputDiffs[i] dscod.Direction = !dscod.Direction cc.DelayedSiacoinOutputDiffs = append(cc.DelayedSiacoinOutputDiffs, dscod) } for i := len(revertedBlock.SiafundPoolDiffs) - 1; i >= 0; i-- { sfpd := revertedBlock.SiafundPoolDiffs[i] sfpd.Direction = modules.DiffRevert cc.SiafundPoolDiffs = append(cc.SiafundPoolDiffs, sfpd) } } for _, appliedBlockID := range ce.AppliedBlocks { appliedBlock, err := getBlockMap(tx, appliedBlockID) if err != nil { cs.log.Critical("getBlockMap failed in computeConsensusChange:", err) return modules.ConsensusChange{}, err } cc.AppliedBlocks = append(cc.AppliedBlocks, appliedBlock.Block) for _, scod := range appliedBlock.SiacoinOutputDiffs { cc.SiacoinOutputDiffs = append(cc.SiacoinOutputDiffs, scod) } for _, fcd := range appliedBlock.FileContractDiffs { cc.FileContractDiffs = append(cc.FileContractDiffs, fcd) } for _, sfod := range appliedBlock.SiafundOutputDiffs { cc.SiafundOutputDiffs = append(cc.SiafundOutputDiffs, sfod) } for _, dscod := range appliedBlock.DelayedSiacoinOutputDiffs { cc.DelayedSiacoinOutputDiffs = append(cc.DelayedSiacoinOutputDiffs, dscod) } for _, sfpd := range appliedBlock.SiafundPoolDiffs { cc.SiafundPoolDiffs = append(cc.SiafundPoolDiffs, sfpd) } } // Grab the child target and the minimum valid child timestamp. recentBlock := ce.AppliedBlocks[len(ce.AppliedBlocks)-1] pb, err := getBlockMap(tx, recentBlock) if err != nil { cs.log.Critical("could not find process block for known block") } cc.ChildTarget = pb.ChildTarget cc.MinimumValidChildTimestamp = cs.blockRuleHelper.minimumValidChildTimestamp(tx.Bucket(BlockMap), pb) currentBlock := currentBlockID(tx) if cs.synced && recentBlock == currentBlock { cc.Synced = true } return cc, nil } // readLockUpdateSubscribers will inform all subscribers of a new update to the // consensus set. readlockUpdateSubscribers does not alter the changelog, the // changelog must be updated beforehand. func (cs *ConsensusSet) readlockUpdateSubscribers(ce changeEntry) { // Get the consensus change and send it to all subscribers. var cc modules.ConsensusChange err := cs.db.View(func(tx *bolt.Tx) error { // Compute the consensus change so it can be sent to subscribers. var err error cc, err = cs.computeConsensusChange(tx, ce) return err }) if err != nil { cs.log.Critical("computeConsensusChange failed:", err) return } for _, subscriber := range cs.subscribers { subscriber.ProcessConsensusChange(cc) } } // initializeSubscribe will take a subscriber and feed them all of the // consensus changes that have occurred since the change provided. // // As a special case, using an empty id as the start will have all the changes // sent to the modules starting with the genesis block. func (cs *ConsensusSet) initializeSubscribe(subscriber modules.ConsensusSetSubscriber, start modules.ConsensusChangeID) error { return cs.db.View(func(tx *bolt.Tx) error { // 'exists' and 'entry' are going to be pointed to the first entry that // has not yet been seen by subscriber. var exists bool var entry changeEntry if start == modules.ConsensusChangeBeginning { // Special case: for modules.ConsensusChangeBeginning, create an // initial node pointing to the genesis block. The subscriber will // receive the diffs for all blocks in the consensus set, including // the genesis block. entry = cs.genesisEntry() exists = true } else if start == modules.ConsensusChangeRecent { // Special case: for modules.ConsensusChangeRecent, set up the // subscriber to start receiving only new blocks, but the // subscriber does not need to do any catch-up. For this // implementation, a no-op will have this effect. return nil } else { // The subscriber has provided an existing consensus change. // Because the subscriber already has this consensus change, // 'entry' and 'exists' need to be pointed at the next consensus // change. entry, exists = getEntry(tx, start) if !exists { // modules.ErrInvalidConsensusChangeID is a named error that // signals a break in synchronization between the consensus set // persistence and the subscriber persistence. Typically, // receiving this error means that the subscriber needs to // perform a rescan of the consensus set. return modules.ErrInvalidConsensusChangeID } entry, exists = entry.NextEntry(tx) } // Send all remaining consensus changes to the subscriber. for exists { cc, err := cs.computeConsensusChange(tx, entry) if err != nil { return err } subscriber.ProcessConsensusChange(cc) entry, exists = entry.NextEntry(tx) } return nil }) } // ConsensusSetSubscribe adds a subscriber to the list of subscribers, and // gives them every consensus change that has occurred since the change with // the provided id. // // As a special case, using an empty id as the start will have all the changes // sent to the modules starting with the genesis block. func (cs *ConsensusSet) ConsensusSetSubscribe(subscriber modules.ConsensusSetSubscriber, start modules.ConsensusChangeID) error { err := cs.tg.Add() if err != nil { return err } defer cs.tg.Done() cs.mu.Lock() defer cs.mu.Unlock() // Get the input module caught up to the currenct consnesus set. cs.subscribers = append(cs.subscribers, subscriber) err = cs.initializeSubscribe(subscriber, start) if err != nil { // Remove the subscriber from the set of subscribers. cs.subscribers = cs.subscribers[:len(cs.subscribers)-1] return err } // Only add the module as a subscriber if there was no error. return nil } // Unsubscribe removes a subscriber from the list of subscribers, allowing for // garbage collection and rescanning. If the subscriber is not found in the // subscriber database, no action is taken. func (cs *ConsensusSet) Unsubscribe(subscriber modules.ConsensusSetSubscriber) { if cs.tg.Add() != nil { return } defer cs.tg.Done() cs.mu.Lock() defer cs.mu.Unlock() // Search for the subscriber in the list of subscribers and remove it if // found. for i := range cs.subscribers { if cs.subscribers[i] == subscriber { cs.subscribers = append(cs.subscribers[0:i], cs.subscribers[i+1:]...) break } } } Sia-1.0.4/modules/consensus/subscribe_test.go000066400000000000000000000105301302033070500212570ustar00rootroot00000000000000package consensus import ( "testing" "github.com/NebulousLabs/Sia/modules" ) // mockSubscriber receives and holds changes to the consensus set, remembering // the order in which changes were received. type mockSubscriber struct { updates []modules.ConsensusChange } // newMockSubscriber returns a mockSubscriber that is ready to subscribe to a // consensus set. Currently blank, but can be expanded to support more features // in the future. func newMockSubscriber() mockSubscriber { return mockSubscriber{} } // ProcessConsensusChange adds a consensus change to the mock subscriber. func (ms *mockSubscriber) ProcessConsensusChange(cc modules.ConsensusChange) { ms.updates = append(ms.updates, cc) } // copySub creates and returns a new mock subscriber that has identical // internals to the input mockSubscriber. The copy will not be subscribed to // the consensus set even if the original is. func (ms *mockSubscriber) copySub() (cms mockSubscriber) { cms.updates = make([]modules.ConsensusChange, len(ms.updates)) copy(cms.updates, ms.updates) return cms } // TestInvalidConsensusChangeSubscription checks that the consensus set returns // modules.ErrInvalidConsensusChangeID in the event of a subscriber using an // unrecognized id. func TestInvalidConsensusChangeSubscription(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestInvalidConsensusChangeSubscription") if err != nil { t.Fatal(err) } defer cst.Close() ms := newMockSubscriber() badCCID := modules.ConsensusChangeID{255, 255, 255} err = cst.cs.ConsensusSetSubscribe(&ms, badCCID) if err != modules.ErrInvalidConsensusChangeID { t.Error("consensus set returning the wrong error during an invalid subscription:", err) } cst.cs.mu.Lock() for i := range cst.cs.subscribers { if cst.cs.subscribers[i] == &ms { t.Fatal("subscriber was not removed from subscriber list after an erroneus subscription") } } cst.cs.mu.Unlock() } // TestInvalidToValidSubscription is a regression test. Previously, the // consensus set would not unsubscribe a module if it returned an error during // subscription. When the module resubscribed, the module would be // double-subscribed to the consensus set. func TestInvalidToValidSubscription(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestInvalidToValidSubscription") if err != nil { t.Fatal(err) } defer cst.Close() // Start by performing a bad subscribe. ms := newMockSubscriber() badCCID := modules.ConsensusChangeID{255, 255, 255} err = cst.cs.ConsensusSetSubscribe(&ms, badCCID) if err != modules.ErrInvalidConsensusChangeID { t.Error("consensus set returning the wrong error during an invalid subscription:", err) } // Perform a correct subscribe. err = cst.cs.ConsensusSetSubscribe(&ms, modules.ConsensusChangeBeginning) if err != nil { t.Fatal(err) } // Mine a block and check that the mock subscriber only got a single // consensus change. numPrevUpdates := len(ms.updates) _, err = cst.miner.AddBlock() if err != nil { t.Fatal(err) } if len(ms.updates) != numPrevUpdates+1 { t.Error("subscriber received two consensus changes for a single block") } } // TestUnsubscribe checks that the consensus set correctly unsubscribes a // subscriber if the Unsubscribe call is made. func TestUnsubscribe(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestUnsubscribe") if err != nil { t.Fatal(err) } defer cst.Close() // Subscribe the mock subscriber to the consensus set. ms := newMockSubscriber() err = cst.cs.ConsensusSetSubscribe(&ms, modules.ConsensusChangeBeginning) if err != nil { t.Fatal(err) } // Check that the subscriber is receiving updates. msLen := len(ms.updates) if msLen == 0 { t.Error("mock subscriber is not receiving updates") } _, err = cst.miner.AddBlock() // should cause another update to be sent to the subscriber if err != nil { t.Fatal(err) } if len(ms.updates) != msLen+1 { t.Error("mock subscriber did not receive the correct number of updates") } // Unsubscribe the subscriber and then check that it is no longer receiving // updates. cst.cs.Unsubscribe(&ms) _, err = cst.miner.AddBlock() if err != nil { t.Fatal(err) } if len(ms.updates) != msLen+1 { t.Error("mock subscriber was not correctly unsubscribed") } } Sia-1.0.4/modules/consensus/synchronize.go000066400000000000000000000465421302033070500206260ustar00rootroot00000000000000package consensus import ( "errors" "net" "sync" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) const ( // minNumOutbound is the minimum number of outbound peers required before ibd // is confident we are synced. minNumOutbound = 5 ) var ( // MaxCatchUpBlocks is the maxiumum number of blocks that can be given to // the consensus set in a single iteration during the initial blockchain // download. MaxCatchUpBlocks = func() types.BlockHeight { switch build.Release { case "dev": return 50 case "standard": return 10 case "testing": return 3 default: panic("unrecognized build.Release") } }() // sendBlocksTimeout is the timeout for the SendBlocks RPC. sendBlocksTimeout = func() time.Duration { switch build.Release { case "dev": return 40 * time.Second case "standard": return 5 * time.Minute case "testing": return 5 * time.Second default: panic("unrecognized build.Release") } }() // minIBDWaitTime is the time threadedInitialBlockchainDownload waits before // exiting if there are >= 1 and <= minNumOutbound peers synced. This timeout // will primarily affect miners who have multiple nodes daisy chained off each // other. Those nodes will likely have to wait minIBDWaitTime on every startup // before IBD is done. minIBDWaitTime = func() time.Duration { switch build.Release { case "dev": return 80 * time.Second case "standard": return 90 * time.Minute case "testing": return 10 * time.Second default: panic("unrecognized build.Release") } }() // ibdLoopDelay is the time that threadedInitialBlockchainDownload waits // between attempts to synchronize with the network if the last attempt // failed. ibdLoopDelay = func() time.Duration { switch build.Release { case "dev": return 1 * time.Second case "standard": return 10 * time.Second case "testing": return 100 * time.Millisecond default: panic("unrecognized build.Release") } }() errEarlyStop = errors.New("initial blockchain download did not complete by the time shutdown was issued") errSendBlocksStalled = errors.New("SendBlocks RPC timed and never received any blocks") ) // blockHistory returns up to 32 block ids, starting with recent blocks and // then proving exponentially increasingly less recent blocks. The genesis // block is always included as the last block. This block history can be used // to find a common parent that is reasonably recent, usually the most recent // common parent is found, but always a common parent within a factor of 2 is // found. func blockHistory(tx *bolt.Tx) (blockIDs [32]types.BlockID) { height := blockHeight(tx) step := types.BlockHeight(1) // The final step is to include the genesis block, which is why the final // element is skipped during iteration. for i := 0; i < 31; i++ { // Include the next block. blockID, err := getPath(tx, height) if build.DEBUG && err != nil { panic(err) } blockIDs[i] = blockID // Determine the height of the next block to include and then increase // the step size. The height must be decreased first to prevent // underflow. // // `i >= 9` means that the first 10 blocks will be included, and then // skipping will start. if i >= 9 { step *= 2 } if height <= step { break } height -= step } // Include the genesis block as the last element blockID, err := getPath(tx, 0) if build.DEBUG && err != nil { panic(err) } blockIDs[31] = blockID return blockIDs } // managedReceiveBlocks is the calling end of the SendBlocks RPC, without the // threadgroup wrapping. func (cs *ConsensusSet) managedReceiveBlocks(conn modules.PeerConn) (returnErr error) { // Set a deadline after which SendBlocks will timeout. During IBD, esepcially, // SendBlocks will timeout. This is by design so that IBD switches peers to // prevent any one peer from stalling IBD. err := conn.SetDeadline(time.Now().Add(sendBlocksTimeout)) // Ignore errors returned by SetDeadline if the conn is a pipe in testing. // Pipes do not support Set{,Read,Write}Deadline and should only be used in // testing. if opErr, ok := err.(*net.OpError); ok && opErr.Op == "set" && opErr.Net == "pipe" && build.Release == "testing" { err = nil } if err != nil { return err } stalled := true defer func() { // TODO: Timeout errors returned by muxado do not conform to the net.Error // interface and therefore we cannot check if the error is a timeout using // the Timeout() method. Once muxado issue #14 is resolved change the below // condition to: // if netErr, ok := returnErr.(net.Error); ok && netErr.Timeout() && stalled { ... } if stalled && returnErr != nil && (returnErr.Error() == "Read timeout" || returnErr.Error() == "Write timeout") { returnErr = errSendBlocksStalled } }() // Get blockIDs to send. var history [32]types.BlockID cs.mu.RLock() err = cs.db.View(func(tx *bolt.Tx) error { history = blockHistory(tx) return nil }) cs.mu.RUnlock() if err != nil { return err } // Send the block ids. if err := encoding.WriteObject(conn, history); err != nil { return err } // Broadcast the last block accepted. This functionality is in a defer to // ensure that a block is always broadcast if any blocks are accepted. This // is to stop an attacker from preventing block broadcasts. chainExtended := false defer func() { cs.mu.RLock() synced := cs.synced cs.mu.RUnlock() if chainExtended && synced { // The last block received will be the current block since // managedAcceptBlock only returns nil if a block extends the longest chain. currentBlock := cs.managedCurrentBlock() // COMPATv0.5.1 - broadcast the block to all peers <= v0.5.1 and block header to all peers > v0.5.1 var relayBlockPeers, relayHeaderPeers []modules.Peer for _, p := range cs.gateway.Peers() { if build.VersionCmp(p.Version, "0.5.1") <= 0 { relayBlockPeers = append(relayBlockPeers, p) } else { relayHeaderPeers = append(relayHeaderPeers, p) } } go cs.gateway.Broadcast("RelayBlock", currentBlock, relayBlockPeers) go cs.gateway.Broadcast("RelayHeader", currentBlock.Header(), relayHeaderPeers) } }() // Read blocks off of the wire and add them to the consensus set until // there are no more blocks available. moreAvailable := true for moreAvailable { // Read a slice of blocks from the wire. var newBlocks []types.Block if err := encoding.ReadObject(conn, &newBlocks, uint64(MaxCatchUpBlocks)*types.BlockSizeLimit); err != nil { return err } if err := encoding.ReadObject(conn, &moreAvailable, 1); err != nil { return err } // Integrate the blocks into the consensus set. for _, block := range newBlocks { stalled = false // Call managedAcceptBlock instead of AcceptBlock so as not to broadcast // every block. acceptErr := cs.managedAcceptBlock(block) // Set a flag to indicate that we should broadcast the last block received. if acceptErr == nil { chainExtended = true } // ErrNonExtendingBlock must be ignored until headers-first block // sharing is implemented, block already in database should also be // ignored. if acceptErr == modules.ErrNonExtendingBlock || acceptErr == modules.ErrBlockKnown { acceptErr = nil } if acceptErr != nil { return acceptErr } } } return nil } // threadedReceiveBlocks is the calling end of the SendBlocks RPC. func (cs *ConsensusSet) threadedReceiveBlocks(conn modules.PeerConn) error { err := cs.tg.Add() if err != nil { return err } defer cs.tg.Done() return cs.managedReceiveBlocks(conn) } // rpcSendBlocks is the receiving end of the SendBlocks RPC. It returns a // sequential set of blocks based on the 32 input block IDs. The most recent // known ID is used as the starting point, and up to 'MaxCatchUpBlocks' from // that BlockHeight onwards are returned. It also sends a boolean indicating // whether more blocks are available. func (cs *ConsensusSet) rpcSendBlocks(conn modules.PeerConn) error { err := cs.tg.Add() if err != nil { return err } defer cs.tg.Done() // Read a list of blocks known to the requester and find the most recent // block from the current path. var knownBlocks [32]types.BlockID err = encoding.ReadObject(conn, &knownBlocks, 32*crypto.HashSize) if err != nil { return err } // Find the most recent block from knownBlocks in the current path. found := false var start types.BlockHeight var csHeight types.BlockHeight cs.mu.RLock() err = cs.db.View(func(tx *bolt.Tx) error { csHeight = blockHeight(tx) for _, id := range knownBlocks { pb, err := getBlockMap(tx, id) if err != nil { continue } pathID, err := getPath(tx, pb.Height) if err != nil { continue } if pathID != pb.Block.ID() { continue } if pb.Height == csHeight { break } found = true // Start from the child of the common block. start = pb.Height + 1 break } return nil }) cs.mu.RUnlock() if err != nil { return err } // If no matching blocks are found, or if the caller has all known blocks, // don't send any blocks. if !found { // Send 0 blocks. err = encoding.WriteObject(conn, []types.Block{}) if err != nil { return err } // Indicate that no more blocks are available. return encoding.WriteObject(conn, false) } // Send the caller all of the blocks that they are missing. moreAvailable := true for moreAvailable { // Get the set of blocks to send. var blocks []types.Block cs.mu.RLock() err = cs.db.View(func(tx *bolt.Tx) error { height := blockHeight(tx) for i := start; i <= height && i < start+MaxCatchUpBlocks; i++ { id, err := getPath(tx, i) if build.DEBUG && err != nil { panic(err) } pb, err := getBlockMap(tx, id) if build.DEBUG && err != nil { panic(err) } blocks = append(blocks, pb.Block) } moreAvailable = start+MaxCatchUpBlocks <= height start += MaxCatchUpBlocks return nil }) cs.mu.RUnlock() if err != nil { return err } // Send a set of blocks to the caller + a flag indicating whether more // are available. if err = encoding.WriteObject(conn, blocks); err != nil { return err } if err = encoding.WriteObject(conn, moreAvailable); err != nil { return err } } return nil } // rpcRelayBlock is an RPC that accepts a block from a peer. // COMPATv0.5.1 func (cs *ConsensusSet) rpcRelayBlock(conn modules.PeerConn) error { err := cs.tg.Add() if err != nil { return err } defer cs.tg.Done() // Decode the block from the connection. var b types.Block err = encoding.ReadObject(conn, &b, types.BlockSizeLimit) if err != nil { return err } // Submit the block to the consensus set and broadcast it. err = cs.managedAcceptBlock(b) if err == errOrphan { // If the block is an orphan, try to find the parents. The block // received from the peer is discarded and will be downloaded again if // the parent is found. go func() { err := cs.gateway.RPC(conn.RPCAddr(), "SendBlocks", cs.managedReceiveBlocks) if err != nil { cs.log.Debugln("WARN: failed to get parents of orphan block:", err) } }() } if err != nil { return err } cs.managedBroadcastBlock(b) return nil } // threadedRPCRelayHeader is an RPC that accepts a block header from a peer. func (cs *ConsensusSet) threadedRPCRelayHeader(conn modules.PeerConn) error { err := cs.tg.Add() if err != nil { return err } wg := new(sync.WaitGroup) defer func() { go func() { wg.Wait() cs.tg.Done() }() }() // Decode the block header from the connection. var h types.BlockHeader err = encoding.ReadObject(conn, &h, types.BlockHeaderSize) if err != nil { return err } // Start verification inside of a bolt View tx. cs.mu.RLock() err = cs.db.View(func(tx *bolt.Tx) error { // Do some relatively inexpensive checks to validate the header return cs.validateHeader(boltTxWrapper{tx}, h) }) cs.mu.RUnlock() if err == errOrphan { // If the header is an orphan, try to find the parents. Call needs to // be made in a separate goroutine as execution requires calling an // exported gateway method - threadedRPCRelayHeader was likely called // from an exported gateway function. // // NOTE: In general this is bad design. Rather than recycling other // calls, the whole protocol should have been kept in a single RPC. // Because it is not, we have to do weird threading to prevent // deadlocks, and we also have to be concerned every time the code in // managedReceiveBlocks is adjusted. wg.Add(1) go func() { err := cs.gateway.RPC(conn.RPCAddr(), "SendBlocks", cs.managedReceiveBlocks) if err != nil { cs.log.Debugln("WARN: failed to get parents of orphan header:", err) } wg.Done() }() return nil } else if err != nil { return err } // If the header is valid and extends the heaviest chain, fetch the // corresponding block. Call needs to be made in a separate goroutine // because an exported call to the gateway is used, which is a deadlock // risk given that rpcRelayHeader is called from the gateway. // // NOTE: In general this is bad design. Rather than recycling other calls, // the whole protocol should have been kept in a single RPC. Because it is // not, we have to do weird threading to prevent deadlocks, and we also // have to be concerned every time the code in managedReceiveBlock is // adjusted. wg.Add(1) go func() { err = cs.gateway.RPC(conn.RPCAddr(), "SendBlk", cs.managedReceiveBlock(h.ID())) if err != nil { cs.log.Debugln("WARN: failed to get header's corresponding block:", err) } wg.Done() }() return nil } // rpcSendBlk is an RPC that sends the requested block to the requesting peer. func (cs *ConsensusSet) rpcSendBlk(conn modules.PeerConn) error { err := cs.tg.Add() if err != nil { return err } defer cs.tg.Done() // Decode the block id from the connection. var id types.BlockID err = encoding.ReadObject(conn, &id, crypto.HashSize) if err != nil { return err } // Lookup the corresponding block. var b types.Block cs.mu.RLock() err = cs.db.View(func(tx *bolt.Tx) error { pb, err := getBlockMap(tx, id) if err != nil { return err } b = pb.Block return nil }) cs.mu.RUnlock() if err != nil { return err } // Encode and send the block to the caller. err = encoding.WriteObject(conn, b) if err != nil { return err } return nil } // managedReceiveBlock takes a block id and returns an RPCFunc that requests that // block and then calls AcceptBlock on it. The returned function should be used // as the calling end of the SendBlk RPC. Note that although the function // itself does not do any locking, it is still prefixed with "threaded" because // the function it returns calls the exported method AcceptBlock. func (cs *ConsensusSet) managedReceiveBlock(id types.BlockID) modules.RPCFunc { return func(conn modules.PeerConn) error { if err := encoding.WriteObject(conn, id); err != nil { return err } var block types.Block if err := encoding.ReadObject(conn, &block, types.BlockSizeLimit); err != nil { return err } if err := cs.managedAcceptBlock(block); err != nil { return err } cs.managedBroadcastBlock(block) return nil } } // threadedInitialBlockchainDownload performs the IBD on outbound peers. Blocks // are downloaded from one peer at a time in 5 minute intervals, so as to // prevent any one peer from significantly slowing down IBD. // // NOTE: IBD will succeed right now when each peer has a different blockchain. // The height and the block id of the remote peers' current blocks are not // checked to be the same. This can cause issues if you are connected to // outbound peers <= v0.5.1 that are stalled in IBD. func (cs *ConsensusSet) threadedInitialBlockchainDownload() error { // The consensus set will not recognize IBD as complete until it has enough // peers. After the deadline though, it will recognize the blochchain // download as complete even with only one peer. This deadline is helpful // to local-net setups, where a machine will frequently only have one peer // (and that peer will be another machine on the same local network, but // within the local network at least one peer is connected to the braod // network). deadline := time.Now().Add(minIBDWaitTime) numOutboundSynced := 0 numOutboundNotSynced := 0 for { numOutboundSynced = 0 numOutboundNotSynced = 0 for _, p := range cs.gateway.Peers() { // We only sync on outbound peers at first to make IBD less susceptible to // fast-mining and other attacks, as outbound peers are more difficult to // manipulate. if p.Inbound { continue } // Put the rest of the iteration inside of a thread group. err := func() error { err := cs.tg.Add() if err != nil { return err } defer cs.tg.Done() // Request blocks from the peer. The error returned will only be // 'nil' if there are no more blocks to receive. err = cs.gateway.RPC(p.NetAddress, "SendBlocks", cs.managedReceiveBlocks) if err == nil { numOutboundSynced++ // In this case, 'return nil' is equivalent to skipping to // the next iteration of the loop. return nil } numOutboundNotSynced++ // TODO: Timeout errors returned by muxado do not conform to the net.Error // interface and therefore we cannot check if the error is a timeout using // the Timeout() method. Once muxado issue #14 is resolved change the below // condition to: // if netErr, ok := returnErr.(net.Error); !ok || !netErr.Timeout() { ... } if err.Error() != "Read timeout" && err.Error() != "Write timeout" { cs.log.Printf("WARN: disconnecting from peer %v because IBD failed: %v", p.NetAddress, err) // Disconnect if there is an unexpected error (not a timeout). This // includes errSendBlocksStalled. // // We disconnect so that these peers are removed from gateway.Peers() and // do not prevent us from marking ourselves as fully synced. err := cs.gateway.Disconnect(p.NetAddress) if err != nil { cs.log.Printf("WARN: disconnecting from peer %v failed: %v", p.NetAddress, err) } } return nil }() if err != nil { return err } } // The consensus set is not considered synced until a majority of // outbound peers say that we are synced. If less than 10 minutes have // passed, a minimum of 'minNumOutbound' peers must say that we are // synced, otherwise a 1 vs 0 majority is sufficient. // // This scheme is used to prevent malicious peers from being able to // barricade the sync'd status of the consensus set, and to make sure // that consensus sets behind a firewall with only one peer // (potentially a local peer) are still able to eventually conclude // that they have syncrhonized. Miners and hosts will often have setups // beind a firewall where there is a single node with many peers and // then the rest of the nodes only have a few peers. if numOutboundSynced > numOutboundNotSynced && (numOutboundSynced >= minNumOutbound || time.Now().After(deadline)) { break } else { // Sleep so we don't hammer the network with SendBlock requests. time.Sleep(ibdLoopDelay) } } cs.log.Printf("INFO: IBD done, synced with %v peers", numOutboundSynced) return nil } // Synced returns true if the consensus set is synced with the network. func (cs *ConsensusSet) Synced() bool { err := cs.tg.Add() if err != nil { return false } defer cs.tg.Done() cs.mu.RLock() defer cs.mu.RUnlock() return cs.synced } Sia-1.0.4/modules/consensus/synchronize_ibd_test.go000066400000000000000000000333771302033070500225050ustar00rootroot00000000000000package consensus import ( "errors" "fmt" "io" "net" "path/filepath" "strconv" "sync" "testing" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/gateway" ) // TestSimpleInitialBlockchainDownload tests that // threadedInitialBlockchainDownload synchronizes with peers in the simple case // where there are 8 outbound peers with the same blockchain. func TestSimpleInitialBlockchainDownload(t *testing.T) { if testing.Short() { t.SkipNow() } // Create 8 remote peers. remoteCSTs := make([]*consensusSetTester, 8) for i := range remoteCSTs { cst, err := blankConsensusSetTester(fmt.Sprintf("TestSimpleInitialBlockchainDownload - %v", i)) if err != nil { t.Fatal(err) } defer cst.Close() remoteCSTs[i] = cst } // Create the "local" peer. localCST, err := blankConsensusSetTester("TestSimpleInitialBlockchainDownload - local") if err != nil { t.Fatal(err) } defer localCST.Close() for _, cst := range remoteCSTs { err = localCST.cs.gateway.Connect(cst.cs.gateway.Address()) if err != nil { t.Fatal(err) } } // Give the OnConnectRPCs time to finish. time.Sleep(5 * time.Second) // Test IBD when all peers have only the genesis block. doneChan := make(chan struct{}) go func() { localCST.cs.threadedInitialBlockchainDownload() doneChan <- struct{}{} }() select { case <-doneChan: case <-time.After(5 * time.Second): t.Fatal("initialBlockchainDownload never completed") } if localCST.cs.CurrentBlock().ID() != remoteCSTs[0].cs.CurrentBlock().ID() { t.Fatalf("current block ids do not match: expected '%v', got '%v'", remoteCSTs[0].cs.CurrentBlock().ID(), localCST.cs.CurrentBlock().ID()) } // Test IBD when all remote peers have the same longest chain. for i := 0; i < 20; i++ { b, err := remoteCSTs[0].miner.FindBlock() if err != nil { t.Fatal(err) } for _, cst := range remoteCSTs { err = cst.cs.managedAcceptBlock(b) if err != nil && err != modules.ErrBlockKnown { t.Fatal(err) } } } go func() { localCST.cs.threadedInitialBlockchainDownload() doneChan <- struct{}{} }() select { case <-doneChan: case <-time.After(5 * time.Second): t.Fatal("initialBlockchainDownload never completed") } if localCST.cs.CurrentBlock().ID() != remoteCSTs[0].cs.CurrentBlock().ID() { t.Fatalf("current block ids do not match: expected '%v', got '%v'", remoteCSTs[0].cs.CurrentBlock().ID(), localCST.cs.CurrentBlock().ID()) } // Test IBD when not starting from the genesis block. for i := 0; i < 4; i++ { b, err := remoteCSTs[0].miner.FindBlock() if err != nil { t.Fatal(err) } for _, cst := range remoteCSTs { err = cst.cs.managedAcceptBlock(b) if err != nil && err != modules.ErrBlockKnown { t.Fatal(err) } } } go func() { localCST.cs.threadedInitialBlockchainDownload() doneChan <- struct{}{} }() select { case <-doneChan: case <-time.After(5 * time.Second): t.Fatal("initialBlockchainDownload never completed") } if localCST.cs.CurrentBlock().ID() != remoteCSTs[0].cs.CurrentBlock().ID() { t.Fatalf("current block ids do not match: expected '%v', got '%v'", remoteCSTs[0].cs.CurrentBlock().ID(), localCST.cs.CurrentBlock().ID()) } // Test IBD when the remote peers are on a longer fork. for i := 0; i < 5; i++ { b, err := localCST.miner.FindBlock() if err != nil { t.Fatal(err) } err = localCST.cs.managedAcceptBlock(b) if err != nil { t.Fatal(err) } } for i := 0; i < 10; i++ { b, err := remoteCSTs[0].miner.FindBlock() if err != nil { t.Fatal(err) } for _, cst := range remoteCSTs { err = cst.cs.managedAcceptBlock(b) if err != nil && err != modules.ErrBlockKnown { t.Log(i) t.Fatal(err) } } } go func() { localCST.cs.threadedInitialBlockchainDownload() doneChan <- struct{}{} }() select { case <-doneChan: case <-time.After(5 * time.Second): t.Fatal("initialBlockchainDownload never completed") } if localCST.cs.CurrentBlock().ID() != remoteCSTs[0].cs.CurrentBlock().ID() { t.Fatalf("current block ids do not match: expected '%v', got '%v'", remoteCSTs[0].cs.CurrentBlock().ID(), localCST.cs.CurrentBlock().ID()) } // Test IBD when the remote peers are on a shorter fork. for i := 0; i < 10; i++ { b, err := localCST.miner.FindBlock() if err != nil { t.Fatal(err) } err = localCST.cs.managedAcceptBlock(b) if err != nil { t.Fatal(err) } } for i := 0; i < 5; i++ { b, err := remoteCSTs[0].miner.FindBlock() if err != nil { t.Fatal(err) } for _, cst := range remoteCSTs { err = cst.cs.managedAcceptBlock(b) if err != nil && err != modules.ErrBlockKnown { t.Log(i) t.Fatal(err) } } } localCurrentBlock := localCST.cs.CurrentBlock() go func() { localCST.cs.threadedInitialBlockchainDownload() doneChan <- struct{}{} }() select { case <-doneChan: case <-time.After(5 * time.Second): t.Fatal("initialBlockchainDownload never completed") } if localCST.cs.CurrentBlock().ID() != localCurrentBlock.ID() { t.Fatalf("local was on a longer fork and should not have reorged") } if localCST.cs.CurrentBlock().ID() == remoteCSTs[0].cs.CurrentBlock().ID() { t.Fatalf("ibd syncing is one way, and a longer fork on the local cs should not cause a reorg on the remote cs's") } } type mockGatewayRPCError struct { modules.Gateway rpcErrs map[modules.NetAddress]error mu sync.Mutex } func (g *mockGatewayRPCError) RPC(addr modules.NetAddress, name string, fn modules.RPCFunc) error { g.mu.Lock() defer g.mu.Unlock() return g.rpcErrs[addr] } // TestInitialBlockChainDownloadDisconnects tests that // threadedInitialBlockchainDownload only disconnects from peers that error // with anything but a timeout. func TestInitialBlockchainDownloadDisconnects(t *testing.T) { if testing.Short() { t.SkipNow() } testdir := build.TempDir(modules.ConsensusDir, "TestInitialBlockchainDownloadDisconnects") g, err := gateway.New("localhost:0", false, filepath.Join(testdir, "local", modules.GatewayDir)) if err != nil { t.Fatal(err) } defer g.Close() mg := mockGatewayRPCError{ Gateway: g, rpcErrs: make(map[modules.NetAddress]error), } localCS, err := New(&mg, false, filepath.Join(testdir, "local", modules.ConsensusDir)) if err != nil { t.Fatal(err) } defer localCS.Close() rpcErrs := []error{ // rpcErrs that should cause a a disconnect. io.EOF, errors.New("random error"), errSendBlocksStalled, // rpcErrs that should not cause a disconnect. mockNetError{ error: errors.New("Read timeout"), timeout: true, }, // Need at least minNumOutbound peers that return nil for // threadedInitialBlockchainDownload to mark IBD done. nil, nil, nil, nil, nil, } for i, rpcErr := range rpcErrs { g, err := gateway.New("localhost:0", false, filepath.Join(testdir, "remote - "+strconv.Itoa(i), modules.GatewayDir)) if err != nil { t.Fatal(err) } defer g.Close() err = localCS.gateway.Connect(g.Address()) if err != nil { t.Fatal(err) } mg.rpcErrs[g.Address()] = rpcErr } // Sleep to to give the OnConnectRPCs time to finish. time.Sleep(500 * time.Millisecond) // Do IBD. localCS.threadedInitialBlockchainDownload() // Check that localCS disconnected from peers that errored but did not time out during SendBlocks. if len(localCS.gateway.Peers()) != 6 { t.Error("threadedInitialBlockchainDownload disconnected from peers that timedout or didn't error", len(localCS.gateway.Peers())) } for _, p := range localCS.gateway.Peers() { err = mg.rpcErrs[p.NetAddress] if err == nil { continue } if netErr, ok := err.(net.Error); ok && netErr.Timeout() { continue } t.Fatalf("threadedInitialBlockchainDownload didn't disconnect from a peer that returned '%v', %v", err, p.NetAddress) } } // TestInitialBlockchainDownloadDoneRules tests that // threadedInitialBlockchainDownload only terminates under the appropriate // conditions. Appropriate conditions are: // - at least minNumOutbound synced outbound peers // - or at least 1 synced outbound peer and minIBDWaitTime has passed since beginning IBD. func TestInitialBlockchainDownloadDoneRules(t *testing.T) { if testing.Short() { t.SkipNow() } // Set minIBDWaitTime to 1s for just this test because no blocks are // transferred between peers so the wait time can be very short. actualMinIBDWaitTime := minIBDWaitTime defer func() { minIBDWaitTime = actualMinIBDWaitTime }() minIBDWaitTime = 1 * time.Second testdir := build.TempDir(modules.ConsensusDir, "TestInitialBlockchainDownloadDoneRules") g, err := gateway.New("localhost:0", false, filepath.Join(testdir, "local", modules.GatewayDir)) if err != nil { t.Fatal(err) } defer g.Close() mg := mockGatewayRPCError{ Gateway: g, rpcErrs: make(map[modules.NetAddress]error), } cs, err := New(&mg, false, filepath.Join(testdir, "local", modules.ConsensusDir)) if err != nil { t.Fatal(err) } defer cs.Close() doneChan := make(chan struct{}) // Test when there are 0 peers. go func() { cs.threadedInitialBlockchainDownload() doneChan <- struct{}{} }() defer close(doneChan) select { case <-doneChan: t.Error("threadedInitialBlockchainDownload finished with 0 synced peers") case <-time.After(minIBDWaitTime + ibdLoopDelay): } // Test when there are only inbound peers. Wrap the peers in a function so // that they are closed when the test is finished. func() { inboundCSTs := make([]*consensusSetTester, 8) for i := 0; i < len(inboundCSTs); i++ { inboundCST, err := blankConsensusSetTester(filepath.Join("TestInitialBlockchainDownloadDoneRules", fmt.Sprintf("remote - inbound %v", i))) if err != nil { t.Fatal(err) } defer inboundCST.Close() inboundCST.cs.gateway.Connect(cs.gateway.Address()) } <-doneChan peers := cs.gateway.Peers() outbound := false for _, p := range peers { if !p.Inbound { outbound = true break } } if !outbound { t.Error("threadedInitialBlockchainDownload finished with only inbound peers") } }() // Test when there is 1 outbound peer that isn't synced. go func() { cs.threadedInitialBlockchainDownload() doneChan <- struct{}{} }() gatewayTimesout, err := gateway.New("localhost:0", false, filepath.Join(testdir, "remote - timesout", modules.GatewayDir)) if err != nil { t.Fatal(err) } defer gatewayTimesout.Close() mg.mu.Lock() mg.rpcErrs[gatewayTimesout.Address()] = mockNetError{ error: errors.New("Read timeout"), timeout: true, } mg.mu.Unlock() err = cs.gateway.Connect(gatewayTimesout.Address()) if err != nil { t.Fatal(err) } select { case <-doneChan: t.Error("threadedInitialBlockchainDownload finished with 0 synced peers") case <-time.After(minIBDWaitTime + ibdLoopDelay): } // Test when there is 1 peer that is synced and one that is not synced. gatewayNoTimeout, err := gateway.New("localhost:0", false, filepath.Join(testdir, "remote - no timeout", modules.GatewayDir)) if err != nil { t.Fatal(err) } defer gatewayNoTimeout.Close() mg.mu.Lock() mg.rpcErrs[gatewayNoTimeout.Address()] = nil mg.mu.Unlock() err = cs.gateway.Connect(gatewayNoTimeout.Address()) if err != nil { t.Fatal(err) } select { case <-doneChan: t.Fatal("threadedInitialBlockchainDownload finished with 1 synced peer and 1 non-synced peer") case <-time.After(minIBDWaitTime + ibdLoopDelay): } // Test when there is 2 peers that are synced and one that is not synced. gatewayNoTimeout2, err := gateway.New("localhost:0", false, filepath.Join(testdir, "remote - no timeout", modules.GatewayDir)) if err != nil { t.Fatal(err) } defer gatewayNoTimeout2.Close() mg.mu.Lock() mg.rpcErrs[gatewayNoTimeout2.Address()] = nil mg.mu.Unlock() err = cs.gateway.Connect(gatewayNoTimeout2.Address()) if err != nil { t.Fatal(err) } select { case <-doneChan: case <-time.After(minIBDWaitTime + ibdLoopDelay*2): t.Fatal("threadedInitialBlockchainDownload never finished with 2 synced peers and 1 non-synced peer") } // Test when there are >= minNumOutbound peers and >= minNumOutbound peers are synced. gatewayNoTimeouts := make([]modules.Gateway, minNumOutbound-1) for i := 0; i < len(gatewayNoTimeouts); i++ { tmpG, err := gateway.New("localhost:0", false, filepath.Join(testdir, fmt.Sprintf("remote - no timeout %v", i), modules.GatewayDir)) if err != nil { t.Fatal(err) } defer tmpG.Close() mg.mu.Lock() mg.rpcErrs[tmpG.Address()] = nil mg.mu.Unlock() gatewayNoTimeouts[i] = tmpG err = cs.gateway.Connect(gatewayNoTimeouts[i].Address()) if err != nil { t.Fatal(err) } } go func() { cs.threadedInitialBlockchainDownload() doneChan <- struct{}{} }() select { case <-doneChan: case <-time.After(minIBDWaitTime): t.Fatal("threadedInitialBlockchainDownload didn't finish in less than minIBDWaitTime") } } // TestGenesisBlockSync is a regression test that checks what happens when two // consensus sets with only the genesis block are connected. They should // determine that they are sync'd, however previously they would not sync to // eachother as they would report EOF instead of performing correct block // exchange. func TestGenesisBlockSync(t *testing.T) { if testing.Short() { t.SkipNow() } // Create two consensus sets that have zero blocks each (except for the // genesis block). cst1, err := blankConsensusSetTester("TestGenesisBlockSync1") if err != nil { t.Fatal(err) } cst2, err := blankConsensusSetTester("TestGenesisBlockSync2") if err != nil { t.Fatal(err) } // Connect them. err = cst1.gateway.Connect(cst2.gateway.Address()) if err != nil { t.Fatal(err) } // Block until both report that they are sync'd. for i := 0; i < 100; i++ { time.Sleep(time.Millisecond * 100) if cst1.cs.Synced() && cst2.cs.Synced() { break } } if !cst1.cs.Synced() || !cst2.cs.Synced() { t.Error("Consensus sets did not synchronize to eachother", cst1.cs.Synced(), cst2.cs.Synced()) } time.Sleep(time.Second * 12) if len(cst1.gateway.Peers()) == 0 { t.Error("disconnection occured!") } } Sia-1.0.4/modules/consensus/synchronize_test.go000066400000000000000000001243561302033070500216650ustar00rootroot00000000000000package consensus import ( "errors" "fmt" "io" "net" "path/filepath" "strconv" "sync" "testing" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // TestSynchronize tests that the consensus set can successfully synchronize // to a peer. func TestSynchronize(t *testing.T) { if testing.Short() { t.SkipNow() } cst1, err := createConsensusSetTester("TestSynchronize1") if err != nil { t.Fatal(err) } defer cst1.Close() cst2, err := createConsensusSetTester("TestSynchronize2") if err != nil { t.Fatal(err) } defer cst2.Close() // mine on cst2 until it is above cst1 for cst1.cs.dbBlockHeight() >= cst2.cs.dbBlockHeight() { b, _ := cst2.miner.FindBlock() err = cst2.cs.AcceptBlock(b) if err != nil { t.Fatal(err) } } // connect gateways, triggering a Synchronize err = cst1.gateway.Connect(cst2.gateway.Address()) if err != nil { t.Fatal(err) } // blockchains should now match for i := 0; i < 50; i++ { if cst1.cs.dbCurrentBlockID() != cst2.cs.dbCurrentBlockID() { time.Sleep(250 * time.Millisecond) } } if cst1.cs.dbCurrentBlockID() != cst2.cs.dbCurrentBlockID() { t.Fatal("Synchronize failed") } // Mine on cst2 until it is more than 'MaxCatchUpBlocks' ahead of cst1. // NOTE: we have to disconnect prior to this, otherwise cst2 will relay // blocks to cst1. err = cst1.gateway.Disconnect(cst2.gateway.Address()) if err != nil { t.Fatal(err) } for cst2.cs.dbBlockHeight() < cst1.cs.dbBlockHeight()+3+MaxCatchUpBlocks { b, _ := cst2.miner.FindBlock() err = cst2.cs.AcceptBlock(b) if err != nil { t.Fatal(err) } } // reconnect err = cst1.gateway.Connect(cst2.gateway.Address()) if err != nil { t.Fatal(err) } // block heights should now match for i := 0; i < 50; i++ { if cst1.cs.dbBlockHeight() != cst2.cs.dbBlockHeight() { time.Sleep(250 * time.Millisecond) } } if cst1.cs.dbBlockHeight() != cst2.cs.dbBlockHeight() { t.Fatal("synchronize failed") } // extend cst2 with a "bad" (old) block, and synchronize. cst1 should // reject the bad block. cst2.cs.mu.Lock() id, err := cst2.cs.dbGetPath(0) if err != nil { t.Fatal(err) } cst2.cs.dbPushPath(id) cst2.cs.mu.Unlock() // Sleep for a few seconds to allow the network call between the two time // to occur. time.Sleep(5 * time.Second) if cst1.cs.dbBlockHeight() == cst2.cs.dbBlockHeight() { t.Fatal("cst1 did not reject bad block") } } // TestBlockHistory tests that blockHistory returns the expected sequence of // block IDs. func TestBlockHistory(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestBlockHistory") if err != nil { t.Fatal(err) } defer cst.Close() // mine until we have enough blocks to test blockHistory for cst.cs.dbBlockHeight() < 50 { b, _ := cst.miner.FindBlock() err = cst.cs.AcceptBlock(b) if err != nil { t.Fatal(err) } } var history [32]types.BlockID _ = cst.cs.db.View(func(tx *bolt.Tx) error { history = blockHistory(tx) return nil }) // validate history cst.cs.mu.Lock() // first 10 IDs are linear for i := types.BlockHeight(0); i < 10; i++ { id, err := cst.cs.dbGetPath(cst.cs.dbBlockHeight() - i) if err != nil { t.Fatal(err) } if history[i] != id { t.Errorf("Wrong ID in history: expected %v, got %v", id, history[i]) } } // next 4 IDs are exponential heights := []types.BlockHeight{11, 15, 23, 39} for i, height := range heights { id, err := cst.cs.dbGetPath(cst.cs.dbBlockHeight() - height) if err != nil { t.Fatal(err) } if history[10+i] != id { t.Errorf("Wrong ID in history: expected %v, got %v", height, history[10+i]) } } // finally, the genesis ID genesisID, err := cst.cs.dbGetPath(0) if err != nil { t.Fatal(err) } if history[31] != genesisID { t.Errorf("Wrong ID in history: expected %v, got %v", genesisID, history[31]) } cst.cs.mu.Unlock() // remaining IDs should be empty var emptyID types.BlockID for i, id := range history[14:31] { if id != emptyID { t.Errorf("Expected empty ID at index %v, got %v", i+17, id) } } } // mockGatewayCountBroadcasts implements modules.Gateway to mock the Broadcast // method. type mockGatewayCountBroadcasts struct { modules.Gateway numBroadcasts int mu sync.RWMutex } // Broadcast is a mock implementation of modules.Gateway.Broadcast that // increments a counter denoting the number of times it's been called. func (g *mockGatewayCountBroadcasts) Broadcast(name string, obj interface{}, peers []modules.Peer) { g.mu.Lock() g.numBroadcasts++ g.mu.Unlock() g.Gateway.Broadcast(name, obj, peers) } // TestSendBlocksBroadcastsOnce tests that the SendBlocks RPC call only // Broadcasts one block, no matter how many blocks are sent. In the case 0 // blocks are sent, tests that Broadcast is never called. func TestSendBlocksBroadcastsOnce(t *testing.T) { if testing.Short() { t.SkipNow() } // Setup consensus sets. cst1, err := blankConsensusSetTester("TestSendBlocksBroadcastsOnce1") if err != nil { t.Fatal(err) } defer cst1.Close() cst2, err := blankConsensusSetTester("TestSendBlocksBroadcastsOnce2") if err != nil { t.Fatal(err) } defer cst2.Close() // Setup mock gateway. mg := mockGatewayCountBroadcasts{Gateway: cst1.cs.gateway} cst1.cs.gateway = &mg err = cst1.cs.gateway.Connect(cst2.cs.gateway.Address()) if err != nil { t.Fatal(err) } tests := []struct { blocksToMine int expectedNumBroadcasts int synced bool }{ // Test that no blocks are broadcast during IBD. { blocksToMine: 0, expectedNumBroadcasts: 0, synced: false, }, { blocksToMine: 1, expectedNumBroadcasts: 0, synced: false, }, { blocksToMine: 2, expectedNumBroadcasts: 0, synced: false, }, // Test that only one blocks is broadcast when IBD is done. { blocksToMine: 0, expectedNumBroadcasts: 0, synced: true, }, { blocksToMine: 1, expectedNumBroadcasts: 2, synced: true, }, { blocksToMine: 2, expectedNumBroadcasts: 2, synced: true, }, { blocksToMine: int(MaxCatchUpBlocks), expectedNumBroadcasts: 2, synced: true, }, { blocksToMine: 2 * int(MaxCatchUpBlocks), expectedNumBroadcasts: 2, synced: true, }, { blocksToMine: 2*int(MaxCatchUpBlocks) + 1, expectedNumBroadcasts: 2, synced: true, }, } for _, test := range tests { cst1.cs.mu.Lock() cst1.cs.synced = test.synced cst1.cs.mu.Unlock() mg.mu.Lock() mg.numBroadcasts = 0 mg.mu.Unlock() for i := 0; i < test.blocksToMine; i++ { b, minerErr := cst2.miner.FindBlock() if minerErr != nil { t.Fatal(minerErr) } // managedAcceptBlock is used here instead of AcceptBlock so as not to // call Broadcast outside of the SendBlocks RPC. err = cst2.cs.managedAcceptBlock(b) if err != nil { t.Fatal(err) } } err = cst1.cs.gateway.RPC(cst2.cs.gateway.Address(), "SendBlocks", cst1.cs.threadedReceiveBlocks) if err != nil { t.Fatal(err) } // Sleep to wait for possible calls to Broadcast to complete. We cannot // wait on a channel because we don't know how many times broadcast has // been called. time.Sleep(10 * time.Millisecond) mg.mu.RLock() numBroadcasts := mg.numBroadcasts mg.mu.RUnlock() if numBroadcasts != test.expectedNumBroadcasts { t.Errorf("expected %d number of broadcasts, got %d", test.expectedNumBroadcasts, numBroadcasts) } } } // TestIntegrationRPCSendBlocks tests that the SendBlocks RPC adds blocks to // the consensus set, and that the consensus set catches with the remote peer // and possibly reorgs. func TestIntegrationRPCSendBlocks(t *testing.T) { if testing.Short() { t.SkipNow() } type sendBlocksTest struct { commonBlocksToMine types.BlockHeight localBlocksToMine types.BlockHeight remoteBlocksToMine types.BlockHeight msg string } tests := []sendBlocksTest{ { msg: "SendBlocks shouldn't do anything when both CSs are at the genesis block", }, { commonBlocksToMine: 10, msg: "SendBlocks shouldn't do anything when both CSs are at the same block", }, { commonBlocksToMine: 10, localBlocksToMine: 5, msg: "SendBlocks shouldn't do anything when the remote CS is behind the local CS", }, { commonBlocksToMine: 10, remoteBlocksToMine: 5, msg: "SendBlocks should catch up the local CS to the remote CS when it is behind", }, { remoteBlocksToMine: 10, localBlocksToMine: 5, msg: "SendBlocks should reorg the local CS when the remote CS's chain is longer", }, { commonBlocksToMine: 10, remoteBlocksToMine: 10, localBlocksToMine: 5, msg: "SendBlocks should reorg the local CS when the remote CS's chain is longer", }, { remoteBlocksToMine: MaxCatchUpBlocks - 1, msg: "SendBlocks should catch up when the remote CS is ahead", }, { remoteBlocksToMine: MaxCatchUpBlocks - 1, localBlocksToMine: MaxCatchUpBlocks - 2, msg: "SendBlocks should reorg the local CS when the remote CS's chain is longer", }, { remoteBlocksToMine: MaxCatchUpBlocks, msg: "SendBlocks should catch up when the remote CS is ahead", }, { remoteBlocksToMine: MaxCatchUpBlocks, localBlocksToMine: MaxCatchUpBlocks - 2, msg: "SendBlocks should reorg the local CS when the remote CS's chain is longer", }, { remoteBlocksToMine: MaxCatchUpBlocks + 1, // There was a bug that caused SendBlocks to be one block behind when its peer was ahead by (k * MaxCatchUpBlocks) + 1 blocks. msg: "SendBlocks should catch up when the remote CS is ahead", }, { remoteBlocksToMine: MaxCatchUpBlocks + 1, // There was a bug that caused SendBlocks to be one block behind when its peer was ahead by (k * MaxCatchUpBlocks) + 1 blocks. localBlocksToMine: MaxCatchUpBlocks - 2, msg: "SendBlocks should reorg the local CS when the remote CS's chain is longer", }, { remoteBlocksToMine: 2*MaxCatchUpBlocks + 1, msg: "SendBlocks should catch up when the remote CS is ahead", }, { remoteBlocksToMine: 2*MaxCatchUpBlocks + 1, localBlocksToMine: 2*MaxCatchUpBlocks - 2, msg: "SendBlocks should reorg the local CS when the remote CS's chain is longer", }, { remoteBlocksToMine: 12, msg: "SendBlocks should catch up when the remote CS is ahead", }, { remoteBlocksToMine: 15, msg: "SendBlocks should catch up when the remote CS is ahead", }, { remoteBlocksToMine: 16, msg: "SendBlocks should catch up when the remote CS is ahead", }, { remoteBlocksToMine: 17, msg: "SendBlocks should catch up when the remote CS is ahead", }, { remoteBlocksToMine: 23, msg: "SendBlocks should catch up when the remote CS is ahead", }, { remoteBlocksToMine: 31, msg: "SendBlocks should catch up when the remote CS is ahead", }, { remoteBlocksToMine: 32, msg: "SendBlocks should catch up when the remote CS is ahead", }, { remoteBlocksToMine: 33, msg: "SendBlocks should catch up when the remote CS is ahead", }, } for i := 1; i < 10; i++ { tests = append(tests, sendBlocksTest{ remoteBlocksToMine: types.BlockHeight(i), msg: "SendBlocks should catch up when the remote CS is ahead", }) } for i, tt := range tests { // Create the "remote" peer. remoteCST, err := blankConsensusSetTester(filepath.Join("TestIntegrationRPCSendBlocks - remote", strconv.Itoa(i))) if err != nil { t.Fatal(err) } // Create the "local" peer. localCST, err := blankConsensusSetTester(filepath.Join("TestIntegrationRPCSendBlocks - local", strconv.Itoa(i))) if err != nil { t.Fatal(err) } localCST.cs.gateway.Connect(remoteCST.cs.gateway.Address()) // Wait a second to let the OnConnectRPCs finish time.Sleep(100 * time.Millisecond) // Mine blocks. for i := types.BlockHeight(0); i < tt.commonBlocksToMine; i++ { b, err := remoteCST.miner.FindBlock() if err != nil { t.Fatal(err) } err = remoteCST.cs.managedAcceptBlock(b) if err != nil { t.Fatal(err) } err = localCST.cs.managedAcceptBlock(b) if err != nil { t.Fatal(err) } } for i := types.BlockHeight(0); i < tt.remoteBlocksToMine; i++ { b, err := remoteCST.miner.FindBlock() if err != nil { t.Fatal(err) } err = remoteCST.cs.managedAcceptBlock(b) if err != nil { t.Fatal(err) } } for i := types.BlockHeight(0); i < tt.localBlocksToMine; i++ { b, err := localCST.miner.FindBlock() if err != nil { t.Fatal(err) } err = localCST.cs.managedAcceptBlock(b) if err != nil { t.Fatal(err) } } localCurrentBlockID := localCST.cs.CurrentBlock().ID() remoteCurrentBlockID := remoteCST.cs.CurrentBlock().ID() err = localCST.cs.gateway.RPC(remoteCST.cs.gateway.Address(), "SendBlocks", localCST.cs.threadedReceiveBlocks) if err != nil { t.Error(err) } // Assume that if remoteBlocksToMine is greater than localBlocksToMine, then // the local CS must have received the new blocks (and reorged). if tt.remoteBlocksToMine > tt.localBlocksToMine { // Verify that the remote cs did not change. if remoteCST.cs.CurrentBlock().ID() != remoteCurrentBlockID { t.Errorf("%v: the remote CS is at a different current block than before SendBlocks", tt.msg) } // Verify that the local cs got the new blocks. if localCST.cs.Height() != remoteCST.cs.Height() { t.Errorf("%v: expected height %v, got %v", tt.msg, remoteCST.cs.Height(), localCST.cs.Height()) } if localCST.cs.CurrentBlock().ID() != remoteCST.cs.CurrentBlock().ID() { t.Errorf("%v: remote and local CSTs have different current blocks", tt.msg) } } else { // Verify that the local cs did not change. if localCST.cs.CurrentBlock().ID() != localCurrentBlockID { t.Errorf("%v: the local CS is at a different current block than before SendBlocks", tt.msg) } } // Cleanup. err = localCST.Close() if err != nil { t.Fatal(err) } err = remoteCST.Close() if err != nil { t.Fatal(err) } } } // TestRPCSendBlockSendsOnlyNecessaryBlocks tests that the SendBlocks RPC only // sends blocks that the caller does not have and that are part of the longest // chain. func TestRPCSendBlockSendsOnlyNecessaryBlocks(t *testing.T) { if testing.Short() { t.SkipNow() } // Create the "remote" peer. cst, err := blankConsensusSetTester("TestRPCSendBlockSendsOnlyNecessaryBlocks - remote") if err != nil { t.Fatal(err) } defer cst.Close() // Create the "local" peer. // // We create this peer manually (not using blankConsensusSetTester) so that we // can connect it to the remote peer before calling consensus.New so as to // prevent SendBlocks from triggering on Connect. testdir := build.TempDir(modules.ConsensusDir, "TestRPCSendBlockSendsOnlyNecessaryBlocks - local") g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { t.Fatal(err) } defer g.Close() err = g.Connect(cst.cs.gateway.Address()) if err != nil { t.Fatal(err) } cs, err := New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { t.Fatal(err) } defer cs.Close() // Add a few initial blocks to both consensus sets. These are the blocks we // want to make sure SendBlocks is not sending unnecessarily as both parties // already have them. knownBlocks := make(map[types.BlockID]struct{}) for i := 0; i < 20; i++ { b, err := cst.miner.FindBlock() if err != nil { t.Fatal(err) } err = cst.cs.managedAcceptBlock(b) if err != nil { t.Fatal(err) } err = cs.managedAcceptBlock(b) if err != nil { t.Fatal(err) } knownBlocks[b.ID()] = struct{}{} } // Add a few blocks to only the remote peer and store which blocks we add. addedBlocks := make(map[types.BlockID]struct{}) for i := 0; i < 20; i++ { b, err := cst.miner.FindBlock() if err != nil { t.Fatal(err) } err = cst.cs.managedAcceptBlock(b) if err != nil { t.Fatal(err) } addedBlocks[b.ID()] = struct{}{} } err = cs.gateway.RPC(cst.cs.gateway.Address(), "SendBlocks", func(conn modules.PeerConn) error { // Get blockIDs to send. var history [32]types.BlockID cs.mu.RLock() err := cs.db.View(func(tx *bolt.Tx) error { history = blockHistory(tx) return nil }) cs.mu.RUnlock() if err != nil { return err } // Send the block ids. if err := encoding.WriteObject(conn, history); err != nil { return err } moreAvailable := true for moreAvailable { // Read a slice of blocks from the wire. var newBlocks []types.Block if err := encoding.ReadObject(conn, &newBlocks, uint64(MaxCatchUpBlocks)*types.BlockSizeLimit); err != nil { return err } if err := encoding.ReadObject(conn, &moreAvailable, 1); err != nil { return err } // Check if the block needed to be sent. for _, newB := range newBlocks { _, ok := knownBlocks[newB.ID()] if ok { t.Error("SendBlocks sent an unnecessary block that the caller already had") continue } _, ok = addedBlocks[newB.ID()] if !ok { t.Error("SendBlocks sent an unnecessary block that the caller did not have") } } } return nil }) if err != nil { t.Fatal(err) } } // mock PeerConns for testing peer conns that fail reading or writing. type ( mockPeerConn struct { net.Conn } mockPeerConnFailingReader struct { mockPeerConn } mockPeerConnFailingWriter struct { mockPeerConn } ) var ( errFailingReader = errors.New("failing reader") errFailingWriter = errors.New("failing writer") ) // RPCAddr implements this method of the modules.PeerConn interface. func (pc mockPeerConn) RPCAddr() modules.NetAddress { return "mockPeerConn dialback addr" } // Read is a mock implementation of modules.PeerConn.Read that always returns // an error. func (mockPeerConnFailingReader) Read([]byte) (int, error) { return 0, errFailingReader } // Write is a mock implementation of modules.PeerConn.Write that always returns // an error. func (mockPeerConnFailingWriter) Write([]byte) (int, error) { return 0, errFailingWriter } // TestSendBlk probes the ConsensusSet.rpcSendBlk method and tests that it // correctly receives block ids and writes out the corresponding blocks. func TestSendBlk(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := blankConsensusSetTester("TestSendBlk") if err != nil { t.Fatal(err) } defer cst.Close() p1, p2 := net.Pipe() mockP1 := mockPeerConn{p1} fnErr := make(chan error) tests := []struct { id types.BlockID conn modules.PeerConn fn func() // handle reading and writing over the pipe to the mock conn. errWant error msg string }{ // TODO: Test with a failing database. // Test with a failing reader. { conn: mockPeerConnFailingReader{mockP1}, fn: func() { fnErr <- nil }, errWant: errFailingReader, msg: "expected rpcSendBlk to error with a failing reader conn", }, // Test with a block id not found in the blockmap. { conn: mockP1, fn: func() { // Write a block id to the conn. fnErr <- encoding.WriteObject(p2, types.BlockID{}) }, errWant: errNilItem, msg: "expected rpcSendBlk to error with a nonexistent block id", }, // Test with a failing writer. { conn: mockPeerConnFailingWriter{mockP1}, fn: func() { // Write a valid block id to the conn. fnErr <- encoding.WriteObject(p2, types.GenesisID) }, errWant: errFailingWriter, msg: "expected rpcSendBlk to error with a failing writer conn", }, // Test with a valid conn and valid block. { conn: mockP1, fn: func() { // Write a valid block id to the conn. if err := encoding.WriteObject(p2, types.GenesisID); err != nil { fnErr <- err } // Read the block written to the conn. var block types.Block if err := encoding.ReadObject(p2, &block, types.BlockSizeLimit); err != nil { fnErr <- err } // Verify the block is the expected block. if block.ID() != types.GenesisID { fnErr <- fmt.Errorf("rpcSendBlk wrote a different block to conn than the block requested. requested block id: %v, received block id: %v", types.GenesisID, block.ID()) } fnErr <- nil }, errWant: nil, msg: "expected rpcSendBlk to succeed with a valid conn and valid block", }, } for _, tt := range tests { go tt.fn() err := cst.cs.rpcSendBlk(tt.conn) if err != tt.errWant { t.Errorf("%s: expected to fail with `%v', got: `%v'", tt.msg, tt.errWant, err) } err = <-fnErr if err != nil { t.Fatal(err) } } } // TestThreadedReceiveBlock probes the RPCFunc returned by // cs.threadedReceiveBlock and tests that it correctly requests a block id and // receives a block. Also tests that the block is correctly (not) accepted into // the consensus set. func TestThreadedReceiveBlock(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := blankConsensusSetTester("TestThreadedReceiveBlock") if err != nil { t.Fatal(err) } defer cst.Close() p1, p2 := net.Pipe() mockP1 := mockPeerConn{p1} fnErr := make(chan error) tests := []struct { id types.BlockID conn modules.PeerConn fn func() // handle reading and writing over the pipe to the mock conn. errWant error msg string }{ // Test with failing writer. { conn: mockPeerConnFailingWriter{mockP1}, fn: func() { fnErr <- nil }, errWant: errFailingWriter, msg: "the function returned from threadedReceiveBlock should fail with a PeerConn with a failing writer", }, // Test with failing reader. { conn: mockPeerConnFailingReader{mockP1}, fn: func() { // Read the id written to conn. var id types.BlockID if err := encoding.ReadObject(p2, &id, crypto.HashSize); err != nil { fnErr <- err } // Verify the id is the expected id. expectedID := types.BlockID{} if id != expectedID { fnErr <- fmt.Errorf("id written to conn was %v, but id received was %v", expectedID, id) } fnErr <- nil }, errWant: errFailingReader, msg: "the function returned from threadedReceiveBlock should fail with a PeerConn with a failing reader", }, // Test with a valid conn, but an invalid block. { id: types.BlockID{1}, conn: mockP1, fn: func() { // Read the id written to conn. var id types.BlockID if err := encoding.ReadObject(p2, &id, crypto.HashSize); err != nil { fnErr <- err } // Verify the id is the expected id. expectedID := types.BlockID{1} if id != expectedID { fnErr <- fmt.Errorf("id written to conn was %v, but id received was %v", expectedID, id) } // Write an invalid block to conn. block := types.Block{} if err := encoding.WriteObject(p2, block); err != nil { fnErr <- err } fnErr <- nil }, errWant: errOrphan, msg: "the function returned from threadedReceiveBlock should not accept an invalid block", }, // Test with a valid conn and a valid block. { id: types.BlockID{2}, conn: mockP1, fn: func() { // Read the id written to conn. var id types.BlockID if err := encoding.ReadObject(p2, &id, crypto.HashSize); err != nil { fnErr <- err } // Verify the id is the expected id. expectedID := types.BlockID{2} if id != expectedID { fnErr <- fmt.Errorf("id written to conn was %v, but id received was %v", expectedID, id) } // Write a valid block to conn. block, err := cst.miner.FindBlock() if err != nil { fnErr <- err } if err := encoding.WriteObject(p2, block); err != nil { fnErr <- err } fnErr <- nil }, errWant: nil, msg: "the function returned from manageddReceiveBlock should accept a valid block", }, } for _, tt := range tests { managedReceiveFN := cst.cs.managedReceiveBlock(tt.id) go tt.fn() err := managedReceiveFN(tt.conn) if err != tt.errWant { t.Errorf("%s: expected to fail with `%v', got: `%v'", tt.msg, tt.errWant, err) } err = <-fnErr if err != nil { t.Fatal(err) } } } // TestIntegrationSendBlkRPC probes the SendBlk RPC and tests that blocks are // correctly requested, received, and accepted into the consensus set. func TestIntegrationSendBlkRPC(t *testing.T) { if testing.Short() { t.SkipNow() } cst1, err := blankConsensusSetTester("TestIntegrationSendBlkRPC1") if err != nil { t.Fatal(err) } defer cst1.Close() cst2, err := blankConsensusSetTester("TestIntegrationSendBlkRPC2") if err != nil { t.Fatal(err) } defer cst2.Close() err = cst1.cs.gateway.Connect(cst2.cs.gateway.Address()) if err != nil { t.Fatal(err) } // Sleep to give the consensus sets time to finish the background startup // routines - if the block mined below is mined before the sets finish // synchronizing to each other, it screws up the test. time.Sleep(500 * time.Millisecond) // Test that cst1 doesn't accept a block it's already seen (the genesis block). err = cst1.cs.gateway.RPC(cst2.cs.gateway.Address(), "SendBlk", cst1.cs.managedReceiveBlock(types.GenesisID)) if err != modules.ErrBlockKnown { t.Errorf("cst1 should reject known blocks: expected error '%v', got '%v'", modules.ErrBlockKnown, err) } // Test that cst2 errors when it doesn't recognize the requested block. err = cst1.cs.gateway.RPC(cst2.cs.gateway.Address(), "SendBlk", cst1.cs.managedReceiveBlock(types.BlockID{})) if err != io.EOF { t.Errorf("cst2 shouldn't return a block it doesn't recognize: expected error '%v', got '%v'", io.EOF, err) } // Test that cst1 accepts a block that extends its longest chain. block, err := cst2.miner.FindBlock() if err != nil { t.Fatal(err) } err = cst2.cs.managedAcceptBlock(block) // Call managedAcceptBlock so that the block isn't broadcast. if err != nil { t.Fatal(err) } err = cst1.cs.gateway.RPC(cst2.cs.gateway.Address(), "SendBlk", cst1.cs.managedReceiveBlock(block.ID())) if err != nil { t.Errorf("cst1 should accept a block that extends its longest chain: expected nil error, got '%v'", err) } // Test that cst2 accepts a block that extends its longest chain. block, err = cst1.miner.FindBlock() if err != nil { t.Fatal(err) } err = cst1.cs.managedAcceptBlock(block) // Call managedAcceptBlock so that the block isn't broadcast. if err != nil { t.Fatal(err) } err = cst2.cs.gateway.RPC(cst1.cs.gateway.Address(), "SendBlk", cst2.cs.managedReceiveBlock(block.ID())) if err != nil { t.Errorf("cst2 should accept a block that extends its longest chain: expected nil error, got '%v'", err) } // Test that cst1 doesn't accept an orphan block. block, err = cst2.miner.FindBlock() if err != nil { t.Fatal(err) } err = cst2.cs.managedAcceptBlock(block) // Call managedAcceptBlock so that the block isn't broadcast. if err != nil { t.Fatal(err) } block, err = cst2.miner.FindBlock() if err != nil { t.Fatal(err) } err = cst2.cs.managedAcceptBlock(block) // Call managedAcceptBlock so that the block isn't broadcast. if err != nil { t.Fatal(err) } err = cst1.cs.gateway.RPC(cst2.cs.gateway.Address(), "SendBlk", cst1.cs.managedReceiveBlock(block.ID())) if err != errOrphan { t.Errorf("cst1 should not accept an orphan block: expected error '%v', got '%v'", errOrphan, err) } } type mockGatewayCallsRPC struct { modules.Gateway rpcCalled chan string } func (g *mockGatewayCallsRPC) RPC(addr modules.NetAddress, name string, fn modules.RPCFunc) error { g.rpcCalled <- name return nil } // TestRelayHeader tests that rpcRelayHeader requests the corresponding blocks // to valid headers with known parents, or requests the block history to orphan // headers. func TestRelayHeader(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := blankConsensusSetTester("TestRelayHeader") if err != nil { t.Fatal(err) } defer cst.Close() mg := &mockGatewayCallsRPC{ Gateway: cst.cs.gateway, rpcCalled: make(chan string), } cst.cs.gateway = mg p1, p2 := net.Pipe() mockP2 := mockPeerConn{p2} // Valid block that rpcRelayHeader should accept. validBlock, err := cst.miner.FindBlock() if err != nil { t.Fatal(err) } // A block in the near future that rpcRelayHeader return an error for, but // still request the corresponding block. block, target, err := cst.miner.BlockForWork() if err != nil { t.Fatal(err) } block.Timestamp = types.CurrentTimestamp() + 2 + types.FutureThreshold futureBlock, _ := cst.miner.SolveBlock(block, target) tests := []struct { header types.BlockHeader errWant error errMSG string rpcWant string rpcMSG string }{ // Test that rpcRelayHeader rejects known blocks. { header: types.GenesisBlock.Header(), errWant: modules.ErrBlockKnown, errMSG: "rpcRelayHeader should reject headers to known blocks", }, // Test that rpcRelayHeader requests the parent blocks of orphan headers. { header: types.BlockHeader{}, errWant: nil, errMSG: "rpcRelayHeader should not return an error for orphan headers", rpcWant: "SendBlocks", rpcMSG: "rpcRelayHeader should request blocks when the relayed header is an orphan", }, // Test that rpcRelayHeader accepts a valid header that extends the longest chain. { header: validBlock.Header(), errWant: nil, errMSG: "rpcRelayHeader should accept a valid header", rpcWant: "SendBlk", rpcMSG: "rpcRelayHeader should request the block of a valid header", }, // Test that rpcRelayHeader requests a future, but otherwise valid block. { header: futureBlock.Header(), errWant: nil, errMSG: "rpcRelayHeader should not return an error for a future header", rpcWant: "SendBlk", rpcMSG: "rpcRelayHeader should request the corresponding block to a future, but otherwise valid header", }, } errChan := make(chan error) for _, tt := range tests { go func() { errChan <- encoding.WriteObject(p1, tt.header) }() err = cst.cs.threadedRPCRelayHeader(mockP2) if err != tt.errWant { t.Errorf("%s: expected '%v', got '%v'", tt.errMSG, tt.errWant, err) } err = <-errChan if err != nil { t.Fatal(err) } if tt.rpcWant == "" { select { case rpc := <-mg.rpcCalled: t.Errorf("no RPC call expected, but '%v' was called", rpc) case <-time.After(10 * time.Millisecond): } } else { select { case rpc := <-mg.rpcCalled: if rpc != tt.rpcWant { t.Errorf("%s: expected '%v', got '%v'", tt.rpcMSG, tt.rpcWant, rpc) } case <-time.After(10 * time.Millisecond): t.Errorf("%s: expected '%v', but no RPC was called", tt.rpcMSG, tt.rpcWant) } } } } // TestIntegrationBroadcastRelayHeader checks that broadcasting RelayHeader // causes peers to also broadcast the header (if the block is valid). func TestIntegrationBroadcastRelayHeader(t *testing.T) { if testing.Short() { t.SkipNow() } // Setup consensus sets. cst1, err := blankConsensusSetTester("TestIntegrationBroadcastRelayHeader1") if err != nil { t.Fatal(err) } defer cst1.Close() cst2, err := blankConsensusSetTester("TestIntegrationBroadcastRelayHeader2") if err != nil { t.Fatal(err) } defer cst2.Close() // Setup mock gateway. mg := &mockGatewayDoesBroadcast{ Gateway: cst2.cs.gateway, broadcastCalled: make(chan struct{}), } cst2.cs.gateway = mg err = cst1.cs.gateway.Connect(cst2.cs.gateway.Address()) if err != nil { t.Fatal(err) } // Give time for on connect RPCs to finish. time.Sleep(500 * time.Millisecond) // Test that broadcasting an invalid block header over RelayHeader on cst1.cs // does not result in cst2.cs.gateway receiving a broadcast. cst1.cs.gateway.Broadcast("RelayHeader", types.BlockHeader{}, cst1.cs.gateway.Peers()) select { case <-mg.broadcastCalled: t.Fatal("RelayHeader broadcasted an invalid block header") case <-time.After(500 * time.Millisecond): } // Test that broadcasting a valid block header over RelayHeader on cst1.cs // causes cst2.cs.gateway to receive a broadcast. validBlock, err := cst1.miner.FindBlock() if err != nil { t.Fatal(err) } err = cst1.cs.managedAcceptBlock(validBlock) if err != nil { t.Fatal(err) } cst1.cs.gateway.Broadcast("RelayHeader", validBlock.Header(), cst1.cs.gateway.Peers()) select { case <-mg.broadcastCalled: // Broadcast is called twice, once to broadcast blocks to peers <= v0.5.1 // and once to broadcast block headers to peers > v0.5.1. <-mg.broadcastCalled case <-time.After(500 * time.Millisecond): t.Fatal("RelayHeader didn't broadcast a valid block header") } } // TestIntegrationRelaySynchronize tests that blocks are relayed as they are // accepted and that peers stay synchronized. This test is header/block // broadcast agnostic. When build.Version <= 0.5.1 block relaying will be // tested. When build.Version > 0.5.1 header relaying will be tested. func TestIntegrationRelaySynchronize(t *testing.T) { if testing.Short() { t.SkipNow() } cst1, err := blankConsensusSetTester("TestRelaySynchronize1") if err != nil { t.Fatal(err) } defer cst1.Close() cst2, err := blankConsensusSetTester("TestRelaySynchronize2") if err != nil { t.Fatal(err) } defer cst2.Close() cst3, err := blankConsensusSetTester("TestRelaySynchronize3") if err != nil { t.Fatal(err) } defer cst3.Close() // Connect them like so: cst1 <-> cst2 <-> cst3 err = cst1.gateway.Connect(cst2.gateway.Address()) if err != nil { t.Fatal(err) } err = cst2.gateway.Connect(cst3.gateway.Address()) if err != nil { t.Fatal(err) } // Make sure cst1 is not connected to cst3. cst1.gateway.Disconnect(cst3.gateway.Address()) cst3.gateway.Disconnect(cst1.gateway.Address()) // Spin until the connection calls have completed. for i := 0; i < 100; i++ { time.Sleep(150 * time.Millisecond) if len(cst1.gateway.Peers()) >= 1 && len(cst3.gateway.Peers()) >= 1 { break } } if len(cst1.gateway.Peers()) < 1 || len(cst3.gateway.Peers()) < 1 { t.Fatal("Peer connection has failed.") } // Mine a block on cst1, expecting the block to propagate from cst1 to // cst2, and then to cst3. b1, err := cst1.miner.AddBlock() if err != nil { t.Log(b1.ID()) t.Log(cst1.cs.CurrentBlock().ID()) t.Log(cst2.cs.CurrentBlock().ID()) t.Fatal(err) } // Spin until the block has propagated to cst2. for i := 0; i < 100; i++ { time.Sleep(150 * time.Millisecond) if cst2.cs.CurrentBlock().ID() == b1.ID() { break } } if cst2.cs.CurrentBlock().ID() != b1.ID() { t.Fatal("Block propagation has failed") } // Spin until the block has propagated to cst3. for i := 0; i < 100; i++ { time.Sleep(150 * time.Millisecond) if cst3.cs.CurrentBlock().ID() == b1.ID() { break } } if cst3.cs.CurrentBlock().ID() != b1.ID() { t.Fatal("Block propagation has failed") } // Mine a block on cst2. b2, err := cst2.miner.AddBlock() if err != nil { t.Log(b1.ID()) t.Log(b2.ID()) t.Log(cst2.cs.CurrentBlock().ID()) t.Log(cst3.cs.CurrentBlock().ID()) t.Fatal(err) } // Spin until the block has propagated to cst1. for i := 0; i < 100; i++ { time.Sleep(150 * time.Millisecond) if cst1.cs.CurrentBlock().ID() == b2.ID() { break } } if cst1.cs.CurrentBlock().ID() != b2.ID() { t.Fatal("block propagation has failed") } // Spin until the block has propagated to cst3. for i := 0; i < 100; i++ { time.Sleep(150 * time.Millisecond) if cst3.cs.CurrentBlock().ID() == b2.ID() { break } } if cst3.cs.CurrentBlock().ID() != b2.ID() { t.Fatal("block propagation has failed") } // Mine a block on cst3. b3, err := cst3.miner.AddBlock() if err != nil { t.Log(b1.ID()) t.Log(b2.ID()) t.Log(b3.ID()) t.Log(cst1.cs.CurrentBlock().ID()) t.Log(cst2.cs.CurrentBlock().ID()) t.Log(cst3.cs.CurrentBlock().ID()) t.Fatal(err) } // Spin until the block has propagated to cst1. for i := 0; i < 100; i++ { time.Sleep(150 * time.Millisecond) if cst1.cs.CurrentBlock().ID() == b3.ID() { break } } if cst1.cs.CurrentBlock().ID() != b3.ID() { t.Fatal("block propagation has failed") } // Spin until the block has propagated to cst2. for i := 0; i < 100; i++ { time.Sleep(150 * time.Millisecond) if cst2.cs.CurrentBlock().ID() == b3.ID() { break } } if cst2.cs.CurrentBlock().ID() != b3.ID() { t.Fatal("block propagation has failed") } // Check that cst1 and cst3 are not peers, if they are peers then this test // is invalid because it has failed to be certain that blocks can make // multiple hops. if len(cst1.gateway.Peers()) != 1 || cst1.gateway.Peers()[0].NetAddress == cst3.gateway.Address() { t.Fatal("Test is invalid, cst1 and cst3 have connected to each other") } if len(cst3.gateway.Peers()) != 1 || cst3.gateway.Peers()[0].NetAddress == cst1.gateway.Address() { t.Fatal("Test is invalid, cst3 and cst1 have connected to each other") } } // mockPeerConnMockReadWrite is a mock implementation of modules.PeerConn that // returns fails reading or writing if readErr or writeErr is non-nil, // respectively. type mockPeerConnMockReadWrite struct { modules.PeerConn readErr error writeErr error } // Read is a mock implementation of conn.Read that fails with the mock error if // readErr != nil. func (conn mockPeerConnMockReadWrite) Read(b []byte) (n int, err error) { if conn.readErr != nil { return 0, conn.readErr } return conn.PeerConn.Read(b) } // Write is a mock implementation of conn.Write that fails with the mock error // if writeErr != nil. func (conn mockPeerConnMockReadWrite) Write(b []byte) (n int, err error) { if conn.writeErr != nil { return 0, conn.writeErr } return conn.PeerConn.Write(b) } // mockNetError is a mock net.Error. type mockNetError struct { error timeout bool temporary bool } // Timeout is a mock implementation of net.Error.Timeout. func (err mockNetError) Timeout() bool { return err.timeout } // Temporary is a mock implementation of net.Error.Temporary. func (err mockNetError) Temporary() bool { return err.temporary } // TestThreadedReceiveBlocksStalls tests that threadedReceiveBlocks returns // errSendBlocksStalled when the connection times out before a block is // received. func TestThreadedReceiveBlocksStalls(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := blankConsensusSetTester("TestThreadedReceiveBlocksStalls") if err != nil { t.Fatal(err) } defer cst.Close() p1, p2 := net.Pipe() mockP2 := mockPeerConn{p2} writeTimeoutConn := mockPeerConnMockReadWrite{ PeerConn: mockP2, writeErr: mockNetError{ error: errors.New("Write timeout"), timeout: true, }, } readTimeoutConn := mockPeerConnMockReadWrite{ PeerConn: mockP2, readErr: mockNetError{ error: errors.New("Read timeout"), timeout: true, }, } readNetErrConn := mockPeerConnMockReadWrite{ PeerConn: mockP2, readErr: mockNetError{ error: errors.New("mock read net.Error"), }, } writeNetErrConn := mockPeerConnMockReadWrite{ PeerConn: mockP2, writeErr: mockNetError{ error: errors.New("mock write net.Error"), }, } readErrConn := mockPeerConnMockReadWrite{ PeerConn: mockP2, readErr: errors.New("mock read err"), } writeErrConn := mockPeerConnMockReadWrite{ PeerConn: mockP2, writeErr: errors.New("mock write err"), } // Test that threadedReceiveBlocks errors with errSendBlocksStalled when 0 // blocks have been sent and the conn times out. err = cst.cs.threadedReceiveBlocks(writeTimeoutConn) if err != errSendBlocksStalled { t.Errorf("expected threadedReceiveBlocks to err with \"%v\", got \"%v\"", errSendBlocksStalled, err) } errChan := make(chan error) go func() { var knownBlocks [32]types.BlockID errChan <- encoding.ReadObject(p1, &knownBlocks, 32*crypto.HashSize) }() err = cst.cs.threadedReceiveBlocks(readTimeoutConn) if err != errSendBlocksStalled { t.Errorf("expected threadedReceiveBlocks to err with \"%v\", got \"%v\"", errSendBlocksStalled, err) } err = <-errChan if err != nil { t.Fatal(err) } // Test that threadedReceiveBlocks errors when writing the block history fails. // Test with an error of type net.Error. err = cst.cs.threadedReceiveBlocks(writeNetErrConn) if err != writeNetErrConn.writeErr { t.Errorf("expected threadedReceiveBlocks to err with \"%v\", got \"%v\"", writeNetErrConn.writeErr, err) } // Test with an error of type error. err = cst.cs.threadedReceiveBlocks(writeErrConn) if err != writeErrConn.writeErr { t.Errorf("expected threadedReceiveBlocks to err with \"%v\", got \"%v\"", writeErrConn.writeErr, err) } // Test that threadedReceiveBlocks errors when reading blocks fails. // Test with an error of type net.Error. go func() { var knownBlocks [32]types.BlockID errChan <- encoding.ReadObject(p1, &knownBlocks, 32*crypto.HashSize) }() err = cst.cs.threadedReceiveBlocks(readNetErrConn) if err != readNetErrConn.readErr { t.Errorf("expected threadedReceiveBlocks to err with \"%v\", got \"%v\"", readNetErrConn.readErr, err) } err = <-errChan if err != nil { t.Fatal(err) } // Test with an error of type error. go func() { var knownBlocks [32]types.BlockID errChan <- encoding.ReadObject(p1, &knownBlocks, 32*crypto.HashSize) }() err = cst.cs.threadedReceiveBlocks(readErrConn) if err != readErrConn.readErr { t.Errorf("expected threadedReceiveBlocks to err with \"%v\", got \"%v\"", readErrConn.readErr, err) } err = <-errChan if err != nil { t.Fatal(err) } // TODO: Test that threadedReceiveBlocks doesn't error with a timeout if it has received one block before this timed out read/write. // TODO: Test that threadedReceiveBlocks doesn't error with errSendBlocksStalled if it successfully received one block. } // TestIntegrationSendBlocksStalls tests that the SendBlocks RPC fails with // errSendBlockStalled when the RPC timesout and the requesting end has // received 0 blocks. func TestIntegrationSendBlocksStalls(t *testing.T) { if testing.Short() { t.SkipNow() } cstLocal, err := blankConsensusSetTester("TestThreadedReceiveBlocksTimesout - local") if err != nil { t.Fatal(err) } defer cstLocal.Close() cstRemote, err := blankConsensusSetTester("TestThreadedReceiveBlocksTimesout - remote") if err != nil { t.Fatal(err) } defer cstRemote.Close() cstLocal.cs.gateway.Connect(cstRemote.cs.gateway.Address()) // Lock the remote CST so that SendBlocks blocks and timesout. cstRemote.cs.mu.Lock() defer cstRemote.cs.mu.Unlock() err = cstLocal.cs.gateway.RPC(cstRemote.cs.gateway.Address(), "SendBlocks", cstLocal.cs.threadedReceiveBlocks) if err != errSendBlocksStalled { t.Fatal(err) } } Sia-1.0.4/modules/consensus/validtransaction.go000066400000000000000000000302401302033070500216040ustar00rootroot00000000000000package consensus import ( "errors" "math/big" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) var ( errAlteredRevisionPayouts = errors.New("file contract revision has altered payout volume") errInvalidStorageProof = errors.New("provided storage proof is invalid") errLateRevision = errors.New("file contract revision submitted after deadline") errLowRevisionNumber = errors.New("transaction has a file contract with an outdated revision number") errMissingSiacoinOutput = errors.New("transaction spends a nonexisting siacoin output") errMissingSiafundOutput = errors.New("transaction spends a nonexisting siafund output") errSiacoinInputOutputMismatch = errors.New("siacoin inputs do not equal siacoin outputs for transaction") errSiafundInputOutputMismatch = errors.New("siafund inputs do not equal siafund outputs for transaction") errUnfinishedFileContract = errors.New("file contract window has not yet openend") errUnrecognizedFileContractID = errors.New("cannot fetch storage proof segment for unknown file contract") errWrongUnlockConditions = errors.New("transaction contains incorrect unlock conditions") ) // validSiacoins checks that the siacoin inputs and outputs are valid in the // context of the current consensus set. func validSiacoins(tx *bolt.Tx, t types.Transaction) error { scoBucket := tx.Bucket(SiacoinOutputs) var inputSum types.Currency for _, sci := range t.SiacoinInputs { // Check that the input spends an existing output. scoBytes := scoBucket.Get(sci.ParentID[:]) if scoBytes == nil { return errMissingSiacoinOutput } // Check that the unlock conditions match the required unlock hash. var sco types.SiacoinOutput err := encoding.Unmarshal(scoBytes, &sco) if build.DEBUG && err != nil { panic(err) } if sci.UnlockConditions.UnlockHash() != sco.UnlockHash { return errWrongUnlockConditions } inputSum = inputSum.Add(sco.Value) } if !inputSum.Equals(t.SiacoinOutputSum()) { return errSiacoinInputOutputMismatch } return nil } // storageProofSegment returns the index of the segment that needs to be proven // exists in a file contract. func storageProofSegment(tx *bolt.Tx, fcid types.FileContractID) (uint64, error) { // Check that the parent file contract exists. fcBucket := tx.Bucket(FileContracts) fcBytes := fcBucket.Get(fcid[:]) if fcBytes == nil { return 0, errUnrecognizedFileContractID } // Decode the file contract. var fc types.FileContract err := encoding.Unmarshal(fcBytes, &fc) if build.DEBUG && err != nil { panic(err) } // Get the trigger block id. blockPath := tx.Bucket(BlockPath) triggerHeight := fc.WindowStart - 1 if triggerHeight > blockHeight(tx) { return 0, errUnfinishedFileContract } var triggerID types.BlockID copy(triggerID[:], blockPath.Get(encoding.EncUint64(uint64(triggerHeight)))) // Get the index by appending the file contract ID to the trigger block and // taking the hash, then converting the hash to a numerical value and // modding it against the number of segments in the file. The result is a // random number in range [0, numSegments]. The probability is very // slightly weighted towards the beginning of the file, but because the // size difference between the number of segments and the random number // being modded, the difference is too small to make any practical // difference. seed := crypto.HashAll(triggerID, fcid) numSegments := int64(crypto.CalculateLeaves(fc.FileSize)) seedInt := new(big.Int).SetBytes(seed[:]) index := seedInt.Mod(seedInt, big.NewInt(numSegments)).Uint64() return index, nil } // validStorageProofsPre100e3 runs the code that was running before height // 100e3, which contains a hardforking bug, fixed at block 100e3. // // HARDFORK 100,000 // // Originally, it was impossible to provide a storage proof for data of length // zero. A hardfork was added triggering at block 100,000 to enable an // optimization where hosts could submit empty storage proofs for files of size // 0, saving space on the blockchain in conditions where the renter is content. func validStorageProofs100e3(tx *bolt.Tx, t types.Transaction) error { for _, sp := range t.StorageProofs { // Check that the storage proof itself is valid. segmentIndex, err := storageProofSegment(tx, sp.ParentID) if err != nil { return err } fc, err := getFileContract(tx, sp.ParentID) if err != nil { return err } leaves := crypto.CalculateLeaves(fc.FileSize) segmentLen := uint64(crypto.SegmentSize) if segmentIndex == leaves-1 { segmentLen = fc.FileSize % crypto.SegmentSize } // HARDFORK 21,000 // // Originally, the code used the entire segment to verify the // correctness of the storage proof. This made the code incompatible // with data sizes that did not fill an entire segment. // // This was patched with a hardfork in block 21,000. The new code made // it possible to perform successful storage proofs on the final // segment of a file if the final segment was not crypto.SegmentSize // bytes. // // Unfortunately, a new bug was introduced where storage proofs on the // final segment would fail if the final segment was selected and was // crypto.SegmentSize bytes, because the segmentLen would be set to 0 // instead of crypto.SegmentSize, due to an error with the modulus // math. This new error has been fixed with the block 100,000 hardfork. if (build.Release == "standard" && blockHeight(tx) < 21e3) || (build.Release == "testing" && blockHeight(tx) < 10) { segmentLen = uint64(crypto.SegmentSize) } verified := crypto.VerifySegment( sp.Segment[:segmentLen], sp.HashSet, leaves, segmentIndex, fc.FileMerkleRoot, ) if !verified { return errInvalidStorageProof } } return nil } // validStorageProofs checks that the storage proofs are valid in the context // of the consensus set. func validStorageProofs(tx *bolt.Tx, t types.Transaction) error { if (build.Release == "standard" && blockHeight(tx) < 100e3) || (build.Release == "testing" && blockHeight(tx) < 10) { return validStorageProofs100e3(tx, t) } for _, sp := range t.StorageProofs { // Check that the storage proof itself is valid. segmentIndex, err := storageProofSegment(tx, sp.ParentID) if err != nil { return err } fc, err := getFileContract(tx, sp.ParentID) if err != nil { return err } leaves := crypto.CalculateLeaves(fc.FileSize) segmentLen := uint64(crypto.SegmentSize) // If this segment chosen is the final segment, it should only be as // long as necessary to complete the filesize. if segmentIndex == leaves-1 { segmentLen = fc.FileSize % crypto.SegmentSize } if segmentLen == 0 { segmentLen = uint64(crypto.SegmentSize) } verified := crypto.VerifySegment( sp.Segment[:segmentLen], sp.HashSet, leaves, segmentIndex, fc.FileMerkleRoot, ) if !verified && fc.FileSize > 0 { return errInvalidStorageProof } } return nil } // validFileContractRevision checks that each file contract revision is valid // in the context of the current consensus set. func validFileContractRevisions(tx *bolt.Tx, t types.Transaction) error { for _, fcr := range t.FileContractRevisions { fc, err := getFileContract(tx, fcr.ParentID) if err != nil { return err } // Check that the height is less than fc.WindowStart - revisions are // not allowed to be submitted once the storage proof window has // opened. This reduces complexity for unconfirmed transactions. if blockHeight(tx) > fc.WindowStart { return errLateRevision } // Check that the revision number of the revision is greater than the // revision number of the existing file contract. if fc.RevisionNumber >= fcr.NewRevisionNumber { return errLowRevisionNumber } // Check that the unlock conditions match the unlock hash. if fcr.UnlockConditions.UnlockHash() != fc.UnlockHash { return errWrongUnlockConditions } // Check that the payout of the revision matches the payout of the // original, and that the payouts match each other. var validPayout, missedPayout, oldPayout types.Currency for _, output := range fcr.NewValidProofOutputs { validPayout = validPayout.Add(output.Value) } for _, output := range fcr.NewMissedProofOutputs { missedPayout = missedPayout.Add(output.Value) } for _, output := range fc.ValidProofOutputs { oldPayout = oldPayout.Add(output.Value) } if !validPayout.Equals(oldPayout) { return errAlteredRevisionPayouts } if !missedPayout.Equals(oldPayout) { return errAlteredRevisionPayouts } } return nil } // validSiafunds checks that the siafund portions of the transaction are valid // in the context of the consensus set. func validSiafunds(tx *bolt.Tx, t types.Transaction) (err error) { // Compare the number of input siafunds to the output siafunds. var siafundInputSum types.Currency var siafundOutputSum types.Currency for _, sfi := range t.SiafundInputs { sfo, err := getSiafundOutput(tx, sfi.ParentID) if err != nil { return err } // Check the unlock conditions match the unlock hash. if sfi.UnlockConditions.UnlockHash() != sfo.UnlockHash { return errWrongUnlockConditions } siafundInputSum = siafundInputSum.Add(sfo.Value) } for _, sfo := range t.SiafundOutputs { siafundOutputSum = siafundOutputSum.Add(sfo.Value) } if !siafundOutputSum.Equals(siafundInputSum) { return errSiafundInputOutputMismatch } return } // validTransaction checks that all fields are valid within the current // consensus state. If not an error is returned. func validTransaction(tx *bolt.Tx, t types.Transaction) error { // StandaloneValid will check things like signatures and properties that // should be inherent to the transaction. (storage proof rules, etc.) err := t.StandaloneValid(blockHeight(tx)) if err != nil { return err } // Check that each portion of the transaction is legal given the current // consensus set. err = validSiacoins(tx, t) if err != nil { return err } err = validStorageProofs(tx, t) if err != nil { return err } err = validFileContractRevisions(tx, t) if err != nil { return err } err = validSiafunds(tx, t) if err != nil { return err } return nil } // TryTransactionSet applies the input transactions to the consensus set to // determine if they are valid. An error is returned IFF they are not a valid // set in the current consensus set. The size of the transactions and the set // is not checked. After the transactions have been validated, a consensus // change is returned detailing the diffs that the transaciton set would have. func (cs *ConsensusSet) TryTransactionSet(txns []types.Transaction) (modules.ConsensusChange, error) { err := cs.tg.Add() if err != nil { return modules.ConsensusChange{}, err } defer cs.tg.Done() cs.mu.RLock() defer cs.mu.RUnlock() // applyTransaction will apply the diffs from a transaction and store them // in a block node. diffHolder is the blockNode that tracks the temporary // changes. At the end of the function, all changes that were made to the // consensus set get reverted. diffHolder := new(processedBlock) // Boltdb will only roll back a tx if an error is returned. In the case of // TryTransactionSet, we want to roll back the tx even if there is no // error. So errSuccess is returned. An alternate method would be to // manually manage the tx instead of using 'Update', but that has safety // concerns and is more difficult to implement correctly. errSuccess := errors.New("success") err = cs.db.Update(func(tx *bolt.Tx) error { diffHolder.Height = blockHeight(tx) for _, txn := range txns { err := validTransaction(tx, txn) if err != nil { return err } applyTransaction(tx, diffHolder, txn) } return errSuccess }) if err != errSuccess { return modules.ConsensusChange{}, err } cc := modules.ConsensusChange{ SiacoinOutputDiffs: diffHolder.SiacoinOutputDiffs, FileContractDiffs: diffHolder.FileContractDiffs, SiafundOutputDiffs: diffHolder.SiafundOutputDiffs, DelayedSiacoinOutputDiffs: diffHolder.DelayedSiacoinOutputDiffs, SiafundPoolDiffs: diffHolder.SiafundPoolDiffs, } return cc, nil } Sia-1.0.4/modules/consensus/validtransaction_test.go000066400000000000000000000541531302033070500226540ustar00rootroot00000000000000package consensus import ( "crypto/rand" "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // TestTryValidTransactionSet submits a valid transaction set to the // TryTransactionSet method. func TestTryValidTransactionSet(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestTryValidTransactionSet") if err != nil { t.Fatal(err) } defer cst.Close() initialHash := cst.cs.dbConsensusChecksum() // Try a valid transaction. _, err = cst.wallet.SendSiacoins(types.NewCurrency64(1), types.UnlockHash{}) if err != nil { t.Fatal(err) } txns := cst.tpool.TransactionList() cc, err := cst.cs.TryTransactionSet(txns) if err != nil { t.Error(err) } if cst.cs.dbConsensusChecksum() != initialHash { t.Error("TryTransactionSet did not resotre order") } if len(cc.SiacoinOutputDiffs) == 0 { t.Error("consensus change is missing diffs after verifying a transction clump") } } // TestTryInvalidTransactionSet submits an invalid transaction set to the // TryTransaction method. func TestTryInvalidTransactionSet(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestTryInvalidTransactionSet") if err != nil { t.Fatal(err) } defer cst.Close() initialHash := cst.cs.dbConsensusChecksum() // Try a valid transaction followed by an invalid transaction. _, err = cst.wallet.SendSiacoins(types.NewCurrency64(1), types.UnlockHash{}) if err != nil { t.Fatal(err) } txns := cst.tpool.TransactionList() txn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{}}, } txns = append(txns, txn) cc, err := cst.cs.TryTransactionSet(txns) if err == nil { t.Error("bad transaction survived filter") } if cst.cs.dbConsensusChecksum() != initialHash { t.Error("TryTransactionSet did not restore order") } if len(cc.SiacoinOutputDiffs) != 0 { t.Error("consensus change was not empty despite an error being returned") } } // TestStorageProofBoundaries creates file contracts and submits storage proofs // for them, probing segment boundaries (first segment, last segment, // incomplete segment, etc.). func TestStorageProofBoundaries(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestStorageProofBoundaries") if err != nil { t.Fatal(err) } defer cst.Close() // Mine enough blocks to put us beyond the testing hardfork. for i := 0; i < 10; i++ { _, err = cst.miner.AddBlock() if err != nil { t.Fatal(err) } } // Try storage proofs on data between 0 bytes and 128 bytes (0 segments and // 1 segment). Perform the operation five times because we can't control // which segment gets selected - it is randomly decided by the block. segmentRange := []int{0, 1, 2, 3, 4, 5, 15, 25, 30, 32, 62, 63, 64, 65, 66, 70, 81, 89, 90, 126, 127, 128, 129} for i := 0; i < 3; i++ { randData, err := crypto.RandBytes(140) if err != nil { t.Fatal(err) } // Create a file contract for all sizes of the data between 0 and 2 // segments and put them in the transaction pool. var fcids []types.FileContractID for _, k := range segmentRange { // Create the data and the file contract around it. truncatedData := randData[:k] fc := types.FileContract{ FileSize: uint64(k), FileMerkleRoot: crypto.MerkleRoot(truncatedData), WindowStart: cst.cs.dbBlockHeight() + 2, WindowEnd: cst.cs.dbBlockHeight() + 4, Payout: types.NewCurrency64(500), // Too small to be subject to siafund fee. ValidProofOutputs: []types.SiacoinOutput{{Value: types.NewCurrency64(500)}}, MissedProofOutputs: []types.SiacoinOutput{{Value: types.NewCurrency64(500)}}, } // Create a transaction around the file contract and add it to the // transaction pool. b := cst.wallet.StartTransaction() err = b.FundSiacoins(types.NewCurrency64(500)) if err != nil { t.Fatal(err) } b.AddFileContract(fc) txnSet, err := b.Sign(true) if err != nil { t.Fatal(err) } err = cst.tpool.AcceptTransactionSet(txnSet) if err != nil { t.Fatal(err) } // Store the file contract id for later when building the storage // proof. fcids = append(fcids, txnSet[len(txnSet)-1].FileContractID(0)) } // Mine blocks to get the file contracts into the blockchain and // confirming. for j := 0; j < 2; j++ { _, err = cst.miner.AddBlock() if err != nil { t.Fatal(err) } } // Create storage proofs for the file contracts and submit the proofs // to the blockchain. for j, k := range segmentRange { // Build the storage proof. truncatedData := randData[:k] proofIndex, err := cst.cs.StorageProofSegment(fcids[j]) if err != nil { t.Fatal(err) } base, hashSet := crypto.MerkleProof(truncatedData, proofIndex) sp := types.StorageProof{ ParentID: fcids[j], HashSet: hashSet, } copy(sp.Segment[:], base) if k > 0 { // Try submitting an empty storage proof, to make sure that the // hardfork code didn't accidentally allow empty storage proofs // in situations other than file sizes with 0 bytes. badSP := types.StorageProof{ParentID: fcids[j]} badTxn := types.Transaction{ StorageProofs: []types.StorageProof{badSP}, } if sp.Segment == badSP.Segment { continue } err = cst.tpool.AcceptTransactionSet([]types.Transaction{badTxn}) if err == nil { t.Fatal("An empty storage proof got into the transaction pool with non-empty data") } } // Submit the storage proof to the blockchain in a transaction. txn := types.Transaction{ StorageProofs: []types.StorageProof{sp}, } err = cst.tpool.AcceptTransactionSet([]types.Transaction{txn}) if err != nil { t.Fatal(err, "-", j, k) } } // Mine blocks to get the storage proofs on the blockchain. for j := 0; j < 2; j++ { _, err = cst.miner.AddBlock() if err != nil { t.Fatal(err) } } } } // TestEmptyStorageProof creates file contracts and submits storage proofs for // them, probing segment boundaries (first segment, last segment, incomplete // segment, etc.). func TestEmptyStorageProof(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestEmptyStorageProof") if err != nil { t.Fatal(err) } defer cst.Close() // Mine enough blocks to put us beyond the testing hardfork. for i := 0; i < 10; i++ { _, err = cst.miner.AddBlock() if err != nil { t.Fatal(err) } } // Try storage proofs on data between 0 bytes and 128 bytes (0 segments and // 1 segment). Perform the operation five times because we can't control // which segment gets selected - it is randomly decided by the block. segmentRange := []int{0, 1, 2, 3, 4, 5, 15, 25, 30, 32, 62, 63, 64, 65, 66, 70, 81, 89, 90, 126, 127, 128, 129} for i := 0; i < 3; i++ { randData, err := crypto.RandBytes(140) if err != nil { t.Fatal(err) } // Create a file contract for all sizes of the data between 0 and 2 // segments and put them in the transaction pool. var fcids []types.FileContractID for _, k := range segmentRange { // Create the data and the file contract around it. truncatedData := randData[:k] fc := types.FileContract{ FileSize: uint64(k), FileMerkleRoot: crypto.MerkleRoot(truncatedData), WindowStart: cst.cs.dbBlockHeight() + 2, WindowEnd: cst.cs.dbBlockHeight() + 4, Payout: types.NewCurrency64(500), // Too small to be subject to siafund fee. ValidProofOutputs: []types.SiacoinOutput{{Value: types.NewCurrency64(500)}}, MissedProofOutputs: []types.SiacoinOutput{{Value: types.NewCurrency64(500)}}, } // Create a transaction around the file contract and add it to the // transaction pool. b := cst.wallet.StartTransaction() err = b.FundSiacoins(types.NewCurrency64(500)) if err != nil { t.Fatal(err) } b.AddFileContract(fc) txnSet, err := b.Sign(true) if err != nil { t.Fatal(err) } err = cst.tpool.AcceptTransactionSet(txnSet) if err != nil { t.Fatal(err) } // Store the file contract id for later when building the storage // proof. fcids = append(fcids, txnSet[len(txnSet)-1].FileContractID(0)) } // Mine blocks to get the file contracts into the blockchain and // confirming. for j := 0; j < 2; j++ { _, err = cst.miner.AddBlock() if err != nil { t.Fatal(err) } } // Create storage proofs for the file contracts and submit the proofs // to the blockchain. for j, k := range segmentRange { // Build the storage proof. truncatedData := randData[:k] proofIndex, err := cst.cs.StorageProofSegment(fcids[j]) if err != nil { t.Fatal(err) } base, hashSet := crypto.MerkleProof(truncatedData, proofIndex) sp := types.StorageProof{ ParentID: fcids[j], HashSet: hashSet, } copy(sp.Segment[:], base) // Submit the storage proof to the blockchain in a transaction. txn := types.Transaction{ StorageProofs: []types.StorageProof{sp}, } err = cst.tpool.AcceptTransactionSet([]types.Transaction{txn}) if err != nil { t.Fatal(err, "-", j, k) } } // Mine blocks to get the storage proofs on the blockchain. for j := 0; j < 2; j++ { _, err = cst.miner.AddBlock() if err != nil { t.Fatal(err) } } } } // TestValidSiacoins probes the validSiacoins method of the consensus set. func TestValidSiacoins(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestValidSiacoins") if err != nil { t.Fatal(err) } defer cst.Close() // Create a transaction pointing to a nonexistent siacoin output. txn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{}}, } err = cst.cs.db.View(func(tx *bolt.Tx) error { err := validSiacoins(tx, txn) if err != errMissingSiacoinOutput { t.Fatal(err) } return nil }) if err != nil { t.Fatal(err) } // Create a transaction with invalid unlock conditions. scoid, _, err := cst.cs.getArbSiacoinOutput() if err != nil { t.Fatal(err) } txn = types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: scoid, }}, } err = cst.cs.db.View(func(tx *bolt.Tx) error { err := validSiacoins(tx, txn) if err != errWrongUnlockConditions { t.Fatal(err) } return nil }) if err != nil { t.Fatal(err) } // Create a txn with more outputs than inputs. txn = types.Transaction{ SiacoinOutputs: []types.SiacoinOutput{{ Value: types.NewCurrency64(1), }}, } err = cst.cs.db.View(func(tx *bolt.Tx) error { err := validSiacoins(tx, txn) if err != errSiacoinInputOutputMismatch { t.Fatal(err) } return nil }) if err != nil { t.Fatal(err) } } // TestStorageProofSegment probes the storageProofSegment method of the // consensus set. func TestStorageProofSegment(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestStorageProofSegment") if err != nil { t.Fatal(err) } defer cst.Close() // Submit a file contract that is unrecognized. _, err = cst.cs.dbStorageProofSegment(types.FileContractID{}) if err != errUnrecognizedFileContractID { t.Error(err) } // Try to get the segment of an unfinished file contract. cst.cs.dbAddFileContract(types.FileContractID{}, types.FileContract{ Payout: types.NewCurrency64(1), WindowStart: 100000, }) _, err = cst.cs.dbStorageProofSegment(types.FileContractID{}) if err != errUnfinishedFileContract { t.Error(err) } } // TestValidStorageProofs probes the validStorageProofs method of the consensus // set. func TestValidStorageProofs(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestValidStorageProofs") if err != nil { t.Fatal(err) } defer cst.Close() // COMPATv0.4.0 // // Mine 10 blocks so that the post-hardfork rules are in effect. for i := 0; i < 10; i++ { block, _ := cst.miner.FindBlock() err = cst.cs.AcceptBlock(block) if err != nil { t.Fatal(err) } } // Create a file contract for which a storage proof can be created. var fcid types.FileContractID fcid[0] = 12 simFile := make([]byte, 64*1024) _, err = rand.Read(simFile) if err != nil { t.Fatal(err) } root := crypto.MerkleRoot(simFile) fc := types.FileContract{ FileSize: 64 * 1024, FileMerkleRoot: root, Payout: types.NewCurrency64(1), WindowStart: 2, WindowEnd: 1200, } cst.cs.dbAddFileContract(fcid, fc) // Create a transaction with a storage proof. proofIndex, err := cst.cs.dbStorageProofSegment(fcid) if err != nil { t.Fatal(err) } base, proofSet := crypto.MerkleProof(simFile, proofIndex) txn := types.Transaction{ StorageProofs: []types.StorageProof{{ ParentID: fcid, HashSet: proofSet, }}, } copy(txn.StorageProofs[0].Segment[:], base) err = cst.cs.dbValidStorageProofs(txn) if err != nil { t.Error(err) } // Corrupt the proof set. proofSet[0][0]++ txn = types.Transaction{ StorageProofs: []types.StorageProof{{ ParentID: fcid, HashSet: proofSet, }}, } copy(txn.StorageProofs[0].Segment[:], base) err = cst.cs.dbValidStorageProofs(txn) if err != errInvalidStorageProof { t.Error(err) } // Try to validate a proof for a file contract that doesn't exist. txn.StorageProofs[0].ParentID = types.FileContractID{} err = cst.cs.dbValidStorageProofs(txn) if err != errUnrecognizedFileContractID { t.Error(err) } // Try a proof set where there is padding on the last segment in the file. file := make([]byte, 100) _, err = rand.Read(file) if err != nil { t.Fatal(err) } root = crypto.MerkleRoot(file) fc = types.FileContract{ FileSize: 100, FileMerkleRoot: root, Payout: types.NewCurrency64(1), WindowStart: 2, WindowEnd: 1200, } // Find a proofIndex that has the value '1'. for { fcid[0]++ cst.cs.dbAddFileContract(fcid, fc) proofIndex, err = cst.cs.dbStorageProofSegment(fcid) if err != nil { t.Fatal(err) } if proofIndex == 1 { break } } base, proofSet = crypto.MerkleProof(file, proofIndex) txn = types.Transaction{ StorageProofs: []types.StorageProof{{ ParentID: fcid, HashSet: proofSet, }}, } copy(txn.StorageProofs[0].Segment[:], base) err = cst.cs.dbValidStorageProofs(txn) if err != nil { t.Fatal(err) } } // HARDFORK 21,000 // // TestPreForkValidStorageProofs checks that storage proofs which are invalid // before the hardfork (but valid afterwards) are still rejected before the // hardfork). func TestPreForkValidStorageProofs(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestPreForkValidStorageProofs") if err != nil { t.Fatal(err) } defer cst.Close() // Try a proof set where there is padding on the last segment in the file. file := make([]byte, 100) _, err = rand.Read(file) if err != nil { t.Fatal(err) } root := crypto.MerkleRoot(file) fc := types.FileContract{ FileSize: 100, FileMerkleRoot: root, Payout: types.NewCurrency64(1), WindowStart: 2, WindowEnd: 1200, } // Find a proofIndex that has the value '1'. var fcid types.FileContractID var proofIndex uint64 for { fcid[0]++ cst.cs.dbAddFileContract(fcid, fc) proofIndex, err = cst.cs.dbStorageProofSegment(fcid) if err != nil { t.Fatal(err) } if proofIndex == 1 { break } } base, proofSet := crypto.MerkleProof(file, proofIndex) txn := types.Transaction{ StorageProofs: []types.StorageProof{{ ParentID: fcid, HashSet: proofSet, }}, } copy(txn.StorageProofs[0].Segment[:], base) err = cst.cs.dbValidStorageProofs(txn) if err != errInvalidStorageProof { t.Log(cst.cs.dbBlockHeight()) t.Fatal(err) } } // TestValidFileContractRevisions probes the validFileContractRevisions method // of the consensus set. func TestValidFileContractRevisions(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() cst, err := createConsensusSetTester("TestValidFileContractRevisions") if err != nil { t.Fatal(err) } defer cst.Close() // Grab an address + unlock conditions for the transaction. unlockConditions, err := cst.wallet.NextAddress() if err != nil { t.Fatal(err) } // Create a file contract for which a storage proof can be created. var fcid types.FileContractID fcid[0] = 12 simFile := make([]byte, 64*1024) rand.Read(simFile) root := crypto.MerkleRoot(simFile) fc := types.FileContract{ FileSize: 64 * 1024, FileMerkleRoot: root, WindowStart: 102, WindowEnd: 1200, Payout: types.NewCurrency64(1), UnlockHash: unlockConditions.UnlockHash(), RevisionNumber: 1, } cst.cs.dbAddFileContract(fcid, fc) // Try a working file contract revision. txn := types.Transaction{ FileContractRevisions: []types.FileContractRevision{ { ParentID: fcid, UnlockConditions: unlockConditions, NewRevisionNumber: 2, }, }, } err = cst.cs.dbValidFileContractRevisions(txn) if err != nil { t.Error(err) } // Try a transaction with an insufficient revision number. txn = types.Transaction{ FileContractRevisions: []types.FileContractRevision{ { ParentID: fcid, UnlockConditions: unlockConditions, NewRevisionNumber: 1, }, }, } err = cst.cs.dbValidFileContractRevisions(txn) if err != errLowRevisionNumber { t.Error(err) } txn = types.Transaction{ FileContractRevisions: []types.FileContractRevision{ { ParentID: fcid, UnlockConditions: unlockConditions, NewRevisionNumber: 0, }, }, } err = cst.cs.dbValidFileContractRevisions(txn) if err != errLowRevisionNumber { t.Error(err) } // Submit a file contract revision pointing to an invalid parent. txn.FileContractRevisions[0].ParentID[0]-- err = cst.cs.dbValidFileContractRevisions(txn) if err != errNilItem { t.Error(err) } txn.FileContractRevisions[0].ParentID[0]++ // Submit a file contract revision for a file contract whose window has // already opened. fc, err = cst.cs.dbGetFileContract(fcid) if err != nil { t.Fatal(err) } fc.WindowStart = 0 cst.cs.dbRemoveFileContract(fcid) cst.cs.dbAddFileContract(fcid, fc) txn.FileContractRevisions[0].NewRevisionNumber = 3 err = cst.cs.dbValidFileContractRevisions(txn) if err != errLateRevision { t.Error(err) } // Submit a file contract revision with incorrect unlock conditions. fc.WindowStart = 100 cst.cs.dbRemoveFileContract(fcid) cst.cs.dbAddFileContract(fcid, fc) txn.FileContractRevisions[0].UnlockConditions.Timelock++ err = cst.cs.dbValidFileContractRevisions(txn) if err != errWrongUnlockConditions { t.Error(err) } txn.FileContractRevisions[0].UnlockConditions.Timelock-- // Submit file contract revisions for file contracts with altered payouts. txn.FileContractRevisions[0].NewValidProofOutputs = []types.SiacoinOutput{{ Value: types.NewCurrency64(1), }} txn.FileContractRevisions[0].NewMissedProofOutputs = []types.SiacoinOutput{{ Value: types.NewCurrency64(1), }} err = cst.cs.dbValidFileContractRevisions(txn) if err != errAlteredRevisionPayouts { t.Error(err) } txn.FileContractRevisions[0].NewValidProofOutputs = nil err = cst.cs.dbValidFileContractRevisions(txn) if err != errAlteredRevisionPayouts { t.Error(err) } txn.FileContractRevisions[0].NewValidProofOutputs = []types.SiacoinOutput{{ Value: types.NewCurrency64(1), }} txn.FileContractRevisions[0].NewMissedProofOutputs = nil err = cst.cs.dbValidFileContractRevisions(txn) if err != errAlteredRevisionPayouts { t.Error(err) } } /* // TestValidSiafunds probes the validSiafunds mthod of the consensus set. func TestValidSiafunds(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestValidSiafunds") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a transaction pointing to a nonexistent siafund output. txn := types.Transaction{ SiafundInputs: []types.SiafundInput{{}}, } err = cst.cs.validSiafunds(txn) if err != ErrMissingSiafundOutput { t.Error(err) } // Create a transaction with invalid unlock conditions. var sfoid types.SiafundOutputID cst.cs.db.forEachSiafundOutputs(func(mapSfoid types.SiafundOutputID, sfo types.SiafundOutput) { sfoid = mapSfoid // pointless to do this but I can't think of a better way. }) txn = types.Transaction{ SiafundInputs: []types.SiafundInput{{ ParentID: sfoid, UnlockConditions: types.UnlockConditions{Timelock: 12345}, // avoid collisions with existing outputs }}, } err = cst.cs.validSiafunds(txn) if err != ErrWrongUnlockConditions { t.Error(err) } // Create a transaction with more outputs than inputs. txn = types.Transaction{ SiafundOutputs: []types.SiafundOutput{{ Value: types.NewCurrency64(1), }}, } err = cst.cs.validSiafunds(txn) if err != ErrSiafundInputOutputMismatch { t.Error(err) } } // TestValidTransaction probes the validTransaction method of the consensus // set. func TestValidTransaction(t *testing.T) { if testing.Short() { t.SkipNow() } cst, err := createConsensusSetTester("TestValidTransaction") if err != nil { t.Fatal(err) } defer cst.closeCst() // Create a transaction that is not standalone valid. txn := types.Transaction{ FileContracts: []types.FileContract{{ WindowStart: 0, }}, } err = cst.cs.validTransaction(txn) if err == nil { t.Error("transaction is valid") } // Create a transaction with invalid siacoins. txn = types.Transaction{ SiacoinInputs: []types.SiacoinInput{{}}, } err = cst.cs.validTransaction(txn) if err == nil { t.Error("transaction is valid") } // Create a transaction with invalid storage proofs. txn = types.Transaction{ StorageProofs: []types.StorageProof{{}}, } err = cst.cs.validTransaction(txn) if err == nil { t.Error("transaction is valid") } // Create a transaction with invalid file contract revisions. txn = types.Transaction{ FileContractRevisions: []types.FileContractRevision{{ NewWindowStart: 5000, NewWindowEnd: 5005, ParentID: types.FileContractID{}, }}, } err = cst.cs.validTransaction(txn) if err == nil { t.Error("transaction is valid") } // Create a transaction with invalid siafunds. txn = types.Transaction{ SiafundInputs: []types.SiafundInput{{}}, } err = cst.cs.validTransaction(txn) if err == nil { t.Error("transaction is valid") } } */ Sia-1.0.4/modules/explorer.go000066400000000000000000000106161302033070500160640ustar00rootroot00000000000000package modules import ( "github.com/NebulousLabs/Sia/types" ) const ( // ExplorerDir is the name of the directory that is typically used for the // explorer. ExplorerDir = "explorer" ) type ( // BlockFacts returns a bunch of statistics about the consensus set as they // were at a specific block. BlockFacts struct { BlockID types.BlockID `json:"blockid"` Difficulty types.Currency `json:"difficulty"` EstimatedHashrate types.Currency `json:"estimatedhashrate"` Height types.BlockHeight `json:"height"` MaturityTimestamp types.Timestamp `json:"maturitytimestamp"` Target types.Target `json:"target"` TotalCoins types.Currency `json:"totalcoins"` // Transaction type counts. MinerPayoutCount uint64 `json:"minerpayoutcount"` TransactionCount uint64 `json:"transactioncount"` SiacoinInputCount uint64 `json:"siacoininputcount"` SiacoinOutputCount uint64 `json:"siacoinoutputcount"` FileContractCount uint64 `json:"filecontractcount"` FileContractRevisionCount uint64 `json:"filecontractrevisioncount"` StorageProofCount uint64 `json:"storageproofcount"` SiafundInputCount uint64 `json:"siafundinputcount"` SiafundOutputCount uint64 `json:"siafundoutputcount"` MinerFeeCount uint64 `json:"minerfeecount"` ArbitraryDataCount uint64 `json:"arbitrarydatacount"` TransactionSignatureCount uint64 `json:"transactionsignaturecount"` // Factoids about file contracts. ActiveContractCost types.Currency `json:"activecontractcost"` ActiveContractCount uint64 `json:"activecontractcount"` ActiveContractSize types.Currency `json:"activecontractsize"` TotalContractCost types.Currency `json:"totalcontractcost"` TotalContractSize types.Currency `json:"totalcontractsize"` TotalRevisionVolume types.Currency `json:"totalrevisionvolume"` } // Explorer tracks the blockchain and provides tools for gathering // statistics and finding objects or patterns within the blockchain. Explorer interface { // Block returns the block that matches the input block id. The bool // indicates whether the block appears in the blockchain. Block(types.BlockID) (types.Block, types.BlockHeight, bool) // BlockFacts returns a set of statistics about the blockchain as they // appeared at a given block. BlockFacts(types.BlockHeight) (BlockFacts, bool) // LatestBlockFacts returns the block facts of the last block // in the explorer's database. LatestBlockFacts() BlockFacts // Transaction returns the block that contains the input transaction // id. The transaction itself is either the block (indicating the miner // payouts are somehow involved), or it is a transaction inside of the // block. The bool indicates whether the transaction is found in the // consensus set. Transaction(types.TransactionID) (types.Block, types.BlockHeight, bool) // UnlockHash returns all of the transaction ids associated with the // provided unlock hash. UnlockHash(types.UnlockHash) []types.TransactionID // SiacoinOutput will return the siacoin output associated with the // input id. SiacoinOutput(types.SiacoinOutputID) (types.SiacoinOutput, bool) // SiacoinOutputID returns all of the transaction ids associated with // the provided siacoin output id. SiacoinOutputID(types.SiacoinOutputID) []types.TransactionID // FileContractHistory returns the history associated with a file // contract, which includes the file contract itself and all of the // revisions that have been submitted to the blockchain. The first bool // indicates whether the file contract exists, and the second bool // indicates whether a storage proof was successfully submitted for the // file contract. FileContractHistory(types.FileContractID) (fc types.FileContract, fcrs []types.FileContractRevision, fcExists bool, storageProofExists bool) // FileContractID returns all of the transaction ids associated with // the provided file contract id. FileContractID(types.FileContractID) []types.TransactionID // SiafundOutput will return the siafund output associated with the // input id. SiafundOutput(types.SiafundOutputID) (types.SiafundOutput, bool) // SiafundOutputID returns all of the transaction ids associated with // the provided siafund output id. SiafundOutputID(types.SiafundOutputID) []types.TransactionID Close() error } ) Sia-1.0.4/modules/explorer/000077500000000000000000000000001302033070500155315ustar00rootroot00000000000000Sia-1.0.4/modules/explorer/database.go000066400000000000000000000073561302033070500176370ustar00rootroot00000000000000package explorer import ( "errors" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) var ( errNotExist = errors.New("entry does not exist") // database buckets bucketBlockFacts = []byte("BlockFacts") bucketBlockIDs = []byte("BlockIDs") bucketBlocksDifficulty = []byte("BlocksDifficulty") bucketBlockTargets = []byte("BlockTargets") bucketFileContractHistories = []byte("FileContractHistories") bucketFileContractIDs = []byte("FileContractIDs") bucketSiacoinOutputIDs = []byte("SiacoinOutputIDs") bucketSiacoinOutputs = []byte("SiacoinOutputs") bucketSiafundOutputIDs = []byte("SiafundOutputIDs") bucketSiafundOutputs = []byte("SiafundOutputs") bucketTransactionIDs = []byte("TransactionIDs") bucketUnlockHashes = []byte("UnlockHashes") // bucketInternal is used to store values internal to the explorer bucketInternal = []byte("Internal") // keys for bucketInternal internalBlockHeight = []byte("BlockHeight") internalRecentChange = []byte("RecentChange") ) // These functions all return a 'func(*bolt.Tx) error', which, allows them to // be called concisely with the db.View and db.Update functions, e.g.: // // var height types.BlockHeight // db.View(dbGetAndDecode(bucketBlockIDs, id, &height)) // // Instead of: // // var height types.BlockHeight // db.View(func(tx *bolt.Tx) error { // bytes := tx.Bucket(bucketBlockIDs).Get(encoding.Marshal(id)) // return encoding.Unmarshal(bytes, &height) // }) // dbGetAndDecode returns a 'func(*bolt.Tx) error' that retrieves and decodes // a value from the specified bucket. If the value does not exist, // dbGetAndDecode returns errNotExist. func dbGetAndDecode(bucket []byte, key, val interface{}) func(*bolt.Tx) error { return func(tx *bolt.Tx) error { valBytes := tx.Bucket(bucket).Get(encoding.Marshal(key)) if valBytes == nil { return errNotExist } return encoding.Unmarshal(valBytes, val) } } // dbGetTransactionIDSet returns a 'func(*bolt.Tx) error' that decodes a // bucket of transaction IDs into a slice. If the bucket is nil, // dbGetTransactionIDSet returns errNotExist. func dbGetTransactionIDSet(bucket []byte, key interface{}, ids *[]types.TransactionID) func(*bolt.Tx) error { return func(tx *bolt.Tx) error { b := tx.Bucket(bucket).Bucket(encoding.Marshal(key)) if b == nil { return errNotExist } // decode into a local slice var txids []types.TransactionID err := b.ForEach(func(txid, _ []byte) error { var id types.TransactionID err := encoding.Unmarshal(txid, &id) if err != nil { return err } txids = append(txids, id) return nil }) if err != nil { return err } // set pointer *ids = txids return nil } } // dbGetBlockFacts returns a 'func(*bolt.Tx) error' that decodes // the block facts for `height` into blockfacts func (e *Explorer) dbGetBlockFacts(height types.BlockHeight, bf *blockFacts) func(*bolt.Tx) error { return func(tx *bolt.Tx) error { block, exists := e.cs.BlockAtHeight(height) if !exists { return errors.New("requested block facts for a block that does not exist") } return dbGetAndDecode(bucketBlockFacts, block.ID(), bf)(tx) } } // dbSetInternal sets the specified key of bucketInternal to the encoded value. func dbSetInternal(key []byte, val interface{}) func(*bolt.Tx) error { return func(tx *bolt.Tx) error { return tx.Bucket(bucketInternal).Put(key, encoding.Marshal(val)) } } // dbGetInternal decodes the specified key of bucketInternal into the supplied pointer. func dbGetInternal(key []byte, val interface{}) func(*bolt.Tx) error { return func(tx *bolt.Tx) error { return encoding.Unmarshal(tx.Bucket(bucketInternal).Get(key), val) } } Sia-1.0.4/modules/explorer/explorer.go000066400000000000000000000045051302033070500177240ustar00rootroot00000000000000// The explorer module provides a glimpse into what the Sia network // currently looks like. package explorer import ( "errors" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/types" ) const ( // hashrateEstimationBlocks is the number of blocks that are used to // estimate the current hashrate. hashrateEstimationBlocks = 200 // 33 hours ) var ( errNilCS = errors.New("explorer cannot use a nil consensus set") ) type ( // fileContractHistory stores the original file contract and the chain of // revisions that have affected a file contract through the life of the // blockchain. fileContractHistory struct { Contract types.FileContract Revisions []types.FileContractRevision StorageProof types.StorageProof } // blockFacts contains a set of facts about the consensus set related to a // certain block. The explorer needs some additional information in the // history so that it can calculate certain values, which is one of the // reasons that the explorer uses a separate struct instead of // modules.BlockFacts. blockFacts struct { modules.BlockFacts Timestamp types.Timestamp } // An Explorer contains a more comprehensive view of the blockchain, // including various statistics and metrics. Explorer struct { cs modules.ConsensusSet db *persist.BoltDatabase persistDir string } ) // New creates the internal data structures, and subscribes to // consensus for changes to the blockchain func New(cs modules.ConsensusSet, persistDir string) (*Explorer, error) { // Check that input modules are non-nil if cs == nil { return nil, errNilCS } // Initialize the explorer. e := &Explorer{ cs: cs, persistDir: persistDir, } // Initialize the persistent structures, including the database. err := e.initPersist() if err != nil { return nil, err } // retrieve the current ConsensusChangeID var recentChange modules.ConsensusChangeID err = e.db.View(dbGetInternal(internalRecentChange, &recentChange)) if err != nil { return nil, err } err = cs.ConsensusSetSubscribe(e, recentChange) if err != nil { // TODO: restart from 0 return nil, errors.New("explorer subscription failed: " + err.Error()) } return e, nil } // Close closes the explorer. func (e *Explorer) Close() error { return e.db.Close() } Sia-1.0.4/modules/explorer/explorer_test.go000066400000000000000000000122041302033070500207560ustar00rootroot00000000000000package explorer import ( "path/filepath" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/consensus" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/modules/miner" "github.com/NebulousLabs/Sia/modules/transactionpool" "github.com/NebulousLabs/Sia/modules/wallet" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/types" ) // Explorer tester struct is the helper object for explorer // testing. It holds the helper modules for its testing type explorerTester struct { cs modules.ConsensusSet gateway modules.Gateway miner modules.TestMiner tpool modules.TransactionPool wallet modules.Wallet walletKey crypto.TwofishKey explorer *Explorer testdir string } // createExplorerTester creates a tester object for the explorer module. func createExplorerTester(name string) (*explorerTester, error) { if testing.Short() { panic("createExplorerTester called when in a short test") } // Create and assemble the dependencies. testdir := build.TempDir(modules.ExplorerDir, name) g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, err } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { return nil, err } w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } key, err := crypto.GenerateTwofishKey() if err != nil { return nil, err } _, err = w.Encrypt(key) if err != nil { return nil, err } err = w.Unlock(key) if err != nil { return nil, err } m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.RenterDir)) if err != nil { return nil, err } e, err := New(cs, filepath.Join(testdir, modules.ExplorerDir)) if err != nil { return nil, err } et := &explorerTester{ cs: cs, gateway: g, miner: m, tpool: tp, wallet: w, walletKey: key, explorer: e, testdir: testdir, } // Mine until the wallet has money. for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ { b, _ := et.miner.FindBlock() err = et.cs.AcceptBlock(b) if err != nil { return nil, err } } return et, nil } // reorgToBlank creates a bunch of empty blocks on top of the genesis block // that reorgs the explorer to a state of all blank blocks. func (et *explorerTester) reorgToBlank() error { // Get a unique directory name to house the persistence of the miner // dependencies. dir := et.testdir + " - " + persist.RandomSuffix() // Create a miner and all dependencies to create an alternate chain. g, err := gateway.New("localhost:0", false, filepath.Join(dir, modules.GatewayDir)) if err != nil { return err } cs, err := consensus.New(g, false, filepath.Join(dir, modules.ConsensusDir)) if err != nil { return err } tp, err := transactionpool.New(cs, g, filepath.Join(dir, modules.TransactionPoolDir)) if err != nil { return err } w, err := wallet.New(cs, tp, filepath.Join(dir, modules.WalletDir)) if err != nil { return err } key, err := crypto.GenerateTwofishKey() if err != nil { return err } _, err = w.Encrypt(key) if err != nil { return err } err = w.Unlock(key) if err != nil { return err } m, err := miner.New(cs, tp, w, filepath.Join(dir, modules.RenterDir)) if err != nil { return err } // Mine blocks until the height is higher than the existing consensus, // submitting each block to the explorerTester. currentHeight := cs.Height() for i := types.BlockHeight(0); i <= currentHeight+1; i++ { block, err := m.AddBlock() if err != nil { return err } et.cs.AcceptBlock(block) // error is not checked, will not always be nil } return nil } // TestNilExplorerDependencies tries to initialize an explorer with nil // dependencies, checks that the correct error is returned. func TestNilExplorerDependencies(t *testing.T) { _, err := New(nil, "expdir") if err != errNilCS { t.Fatal("Expecting errNilCS") } } // TestExplorerGenesisHeight checks that when the explorer is initialized and given the // genesis block, the result has the correct height. func TestExplorerGenesisHeight(t *testing.T) { // Create the dependencies. testdir := build.TempDir(modules.HostDir, "TestExplorerGenesisHeight") g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { t.Fatal(err) } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { t.Fatal(err) } // Create the explorer - from the subscription only the genesis block will // be received. e, err := New(cs, testdir) if err != nil { t.Fatal(err) } block, height, exists := e.Block(types.GenesisID) if !exists { t.Error("explorer missing genesis block after initialization") } if block.ID() != types.GenesisID { t.Error("explorer returned wrong genesis block") } if height != 0 { t.Errorf("genesis block hash wrong height: expected 0, got %v", height) } } Sia-1.0.4/modules/explorer/info.go000066400000000000000000000122131302033070500170120ustar00rootroot00000000000000package explorer import ( "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // Block takes a block ID and finds the corresponding block, provided that the // block is in the consensus set. func (e *Explorer) Block(id types.BlockID) (types.Block, types.BlockHeight, bool) { var height types.BlockHeight err := e.db.View(dbGetAndDecode(bucketBlockIDs, id, &height)) if err != nil { return types.Block{}, 0, false } block, exists := e.cs.BlockAtHeight(height) if !exists { return types.Block{}, 0, false } return block, height, true } // BlockFacts returns a set of statistics about the blockchain as they appeared // at a given block height, and a bool indicating whether facts exist for the // given height. func (e *Explorer) BlockFacts(height types.BlockHeight) (modules.BlockFacts, bool) { var bf blockFacts err := e.db.View(e.dbGetBlockFacts(height, &bf)) if err != nil { return modules.BlockFacts{}, false } return bf.BlockFacts, true } // LatestBlockFacts returns a set of statistics about the blockchain as they appeared // at the latest block height in the explorer's consensus set. func (e *Explorer) LatestBlockFacts() modules.BlockFacts { var bf blockFacts err := e.db.View(func(tx *bolt.Tx) error { var height types.BlockHeight err := dbGetInternal(internalBlockHeight, &height)(tx) if err != nil { return err } return e.dbGetBlockFacts(height, &bf)(tx) }) if err != nil { build.Critical(err) } return bf.BlockFacts } // Transaction takes a transaction ID and finds the block containing the // transaction. Because of the miner payouts, the transaction ID might be a // block ID. To find the transaction, iterate through the block. func (e *Explorer) Transaction(id types.TransactionID) (types.Block, types.BlockHeight, bool) { var height types.BlockHeight err := e.db.View(dbGetAndDecode(bucketTransactionIDs, id, &height)) if err != nil { return types.Block{}, 0, false } block, exists := e.cs.BlockAtHeight(height) if !exists { return types.Block{}, 0, false } return block, height, true } // UnlockHash returns the IDs of all the transactions that contain the unlock // hash. An empty set indicates that the unlock hash does not appear in the // blockchain. func (e *Explorer) UnlockHash(uh types.UnlockHash) []types.TransactionID { var ids []types.TransactionID err := e.db.View(dbGetTransactionIDSet(bucketUnlockHashes, uh, &ids)) if err != nil { ids = nil } return ids } // SiacoinOutput returns the siacoin output associated with the specified ID. func (e *Explorer) SiacoinOutput(id types.SiacoinOutputID) (types.SiacoinOutput, bool) { var sco types.SiacoinOutput err := e.db.View(dbGetAndDecode(bucketSiacoinOutputs, id, &sco)) if err != nil { return types.SiacoinOutput{}, false } return sco, true } // SiacoinOutputID returns all of the transactions that contain the specified // siacoin output ID. An empty set indicates that the siacoin output ID does // not appear in the blockchain. func (e *Explorer) SiacoinOutputID(id types.SiacoinOutputID) []types.TransactionID { var ids []types.TransactionID err := e.db.View(dbGetTransactionIDSet(bucketSiacoinOutputIDs, id, &ids)) if err != nil { ids = nil } return ids } // FileContractHistory returns the history associated with the specified file // contract ID, which includes the file contract itself and all of the // revisions that have been submitted to the blockchain. The first bool // indicates whether the file contract exists, and the second bool indicates // whether a storage proof was successfully submitted for the file contract. func (e *Explorer) FileContractHistory(id types.FileContractID) (fc types.FileContract, fcrs []types.FileContractRevision, fcE bool, spE bool) { var history fileContractHistory err := e.db.View(dbGetAndDecode(bucketFileContractHistories, id, &history)) fc = history.Contract fcrs = history.Revisions fcE = err == nil spE = history.StorageProof.ParentID == id return } // FileContractIDs returns all of the transactions that contain the specified // file contract ID. An empty set indicates that the file contract ID does not // appear in the blockchain. func (e *Explorer) FileContractID(id types.FileContractID) []types.TransactionID { var ids []types.TransactionID err := e.db.View(dbGetTransactionIDSet(bucketFileContractIDs, id, &ids)) if err != nil { ids = nil } return ids } // SiafundOutput returns the siafund output associated with the specified ID. func (e *Explorer) SiafundOutput(id types.SiafundOutputID) (types.SiafundOutput, bool) { var sco types.SiafundOutput err := e.db.View(dbGetAndDecode(bucketSiafundOutputs, id, &sco)) if err != nil { return types.SiafundOutput{}, false } return sco, true } // SiafundOutputID returns all of the transactions that contain the specified // siafund output ID. An empty set indicates that the siafund output ID does // not appear in the blockchain. func (e *Explorer) SiafundOutputID(id types.SiafundOutputID) []types.TransactionID { var ids []types.TransactionID err := e.db.View(dbGetTransactionIDSet(bucketSiafundOutputIDs, id, &ids)) if err != nil { ids = nil } return ids } Sia-1.0.4/modules/explorer/info_test.go000066400000000000000000000036451302033070500200620ustar00rootroot00000000000000package explorer import ( "testing" "github.com/NebulousLabs/Sia/types" ) // TestImmediateBlockFacts grabs the block facts object from the block explorer // at the current height and verifies that the data has been filled out. func TestImmediateBlockFacts(t *testing.T) { if testing.Short() { t.SkipNow() } et, err := createExplorerTester("TestImmediateBlockFacts") if err != nil { t.Fatal(err) } facts := et.explorer.LatestBlockFacts() var explorerHeight types.BlockHeight err = et.explorer.db.View(dbGetInternal(internalBlockHeight, &explorerHeight)) if err != nil { t.Fatal(err) } if facts.Height != explorerHeight || explorerHeight == 0 { t.Error("wrong height reported in facts object") } if !facts.TotalCoins.Equals(types.CalculateNumSiacoins(et.cs.Height())) { t.Error("wrong number of total coins:", facts.TotalCoins, et.cs.Height()) } } // TestBlock probes the Block function of the explorer. func TestBlock(t *testing.T) { if testing.Short() { t.SkipNow() } et, err := createExplorerTester("TestBlock") if err != nil { t.Fatal(err) } gb := types.GenesisBlock gbFetch, height, exists := et.explorer.Block(gb.ID()) if !exists || height != 0 || gbFetch.ID() != gb.ID() { t.Error("call to 'Block' inside explorer failed") } } // TestBlockFacts checks that the correct block facts are returned for a query. func TestBlockFacts(t *testing.T) { if testing.Short() { t.SkipNow() } et, err := createExplorerTester("TestBlockFacts") if err != nil { t.Fatal(err) } gb := types.GenesisBlock bf, exists := et.explorer.BlockFacts(0) if !exists || bf.BlockID != gb.ID() || bf.Height != 0 { t.Error("call to 'BlockFacts' inside explorer failed") t.Error("Expecting true ->", exists) t.Error("Expecting", gb.ID(), "->", bf.BlockID) t.Error("Expecting 0 ->", bf.Height) } bf, exists = et.explorer.BlockFacts(1) if !exists || bf.Height != 1 { t.Error("call to 'BlockFacts' has failed") } } Sia-1.0.4/modules/explorer/persist.go000066400000000000000000000033351302033070500175550ustar00rootroot00000000000000package explorer import ( "os" "path/filepath" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) var explorerMetadata = persist.Metadata{ Header: "Sia Explorer", Version: "0.5.2", } // initPersist initializes the persistent structures of the explorer module. func (e *Explorer) initPersist() error { // Make the persist directory err := os.MkdirAll(e.persistDir, 0700) if err != nil { return err } // Open the database db, err := persist.OpenDatabase(explorerMetadata, filepath.Join(e.persistDir, "explorer.db")) if err != nil { return err } e.db = db // Initialize the database err = e.db.Update(func(tx *bolt.Tx) error { buckets := [][]byte{ bucketBlockFacts, bucketBlockIDs, bucketBlocksDifficulty, bucketBlockTargets, bucketFileContractHistories, bucketFileContractIDs, bucketInternal, bucketSiacoinOutputIDs, bucketSiacoinOutputs, bucketSiafundOutputIDs, bucketSiafundOutputs, bucketTransactionIDs, bucketUnlockHashes, } for _, b := range buckets { _, err := tx.CreateBucketIfNotExists(b) if err != nil { return err } } // set default values for the bucketInternal internalDefaults := []struct { key, val []byte }{ {internalBlockHeight, encoding.Marshal(types.BlockHeight(0))}, {internalRecentChange, encoding.Marshal(modules.ConsensusChangeID{})}, } b := tx.Bucket(bucketInternal) for _, d := range internalDefaults { if b.Get(d.key) != nil { continue } err := b.Put(d.key, d.val) if err != nil { return err } } return nil }) if err != nil { return err } return nil } Sia-1.0.4/modules/explorer/update.go000066400000000000000000000450761302033070500173560ustar00rootroot00000000000000package explorer import ( "fmt" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // ProcessConsensusChange follows the most recent changes to the consensus set, // including parsing new blocks and updating the utxo sets. func (e *Explorer) ProcessConsensusChange(cc modules.ConsensusChange) { if len(cc.AppliedBlocks) == 0 { build.Critical("Explorer.ProcessConsensusChange called with a ConsensusChange that has no AppliedBlocks") } err := e.db.Update(func(tx *bolt.Tx) (err error) { // use exception-style error handling to enable more concise update code defer func() { if r := recover(); r != nil { err = fmt.Errorf("%v", r) } }() // get starting block height var blockheight types.BlockHeight err = dbGetInternal(internalBlockHeight, &blockheight)(tx) if err != nil { return err } // Update cumulative stats for reverted blocks. for _, block := range cc.RevertedBlocks { bid := block.ID() tbid := types.TransactionID(bid) blockheight-- dbRemoveBlockID(tx, bid) dbRemoveTransactionID(tx, tbid) // Miner payouts are a transaction target, exists := e.cs.ChildTarget(block.ParentID) if !exists { target = types.RootTarget } dbRemoveBlockTarget(tx, bid, target) // Remove miner payouts for j, payout := range block.MinerPayouts { scoid := block.MinerPayoutID(uint64(j)) dbRemoveSiacoinOutputID(tx, scoid, tbid) dbRemoveUnlockHash(tx, payout.UnlockHash, tbid) } // Remove transactions for _, txn := range block.Transactions { txid := txn.ID() dbRemoveTransactionID(tx, txid) for _, sci := range txn.SiacoinInputs { dbRemoveSiacoinOutputID(tx, sci.ParentID, txid) dbRemoveUnlockHash(tx, sci.UnlockConditions.UnlockHash(), txid) } for k, sco := range txn.SiacoinOutputs { scoid := txn.SiacoinOutputID(uint64(k)) dbRemoveSiacoinOutputID(tx, scoid, txid) dbRemoveUnlockHash(tx, sco.UnlockHash, txid) dbRemoveSiacoinOutput(tx, scoid) } for k, fc := range txn.FileContracts { fcid := txn.FileContractID(uint64(k)) dbRemoveFileContractID(tx, fcid, txid) dbRemoveUnlockHash(tx, fc.UnlockHash, txid) for l, sco := range fc.ValidProofOutputs { scoid := fcid.StorageProofOutputID(types.ProofValid, uint64(l)) dbRemoveSiacoinOutputID(tx, scoid, txid) dbRemoveUnlockHash(tx, sco.UnlockHash, txid) } for l, sco := range fc.MissedProofOutputs { scoid := fcid.StorageProofOutputID(types.ProofMissed, uint64(l)) dbRemoveSiacoinOutputID(tx, scoid, txid) dbRemoveUnlockHash(tx, sco.UnlockHash, txid) } dbRemoveFileContract(tx, fcid) } for _, fcr := range txn.FileContractRevisions { dbRemoveFileContractID(tx, fcr.ParentID, txid) dbRemoveUnlockHash(tx, fcr.UnlockConditions.UnlockHash(), txid) dbRemoveUnlockHash(tx, fcr.NewUnlockHash, txid) for l, sco := range fcr.NewValidProofOutputs { scoid := fcr.ParentID.StorageProofOutputID(types.ProofValid, uint64(l)) dbRemoveSiacoinOutputID(tx, scoid, txid) dbRemoveUnlockHash(tx, sco.UnlockHash, txid) } for l, sco := range fcr.NewMissedProofOutputs { scoid := fcr.ParentID.StorageProofOutputID(types.ProofMissed, uint64(l)) dbRemoveSiacoinOutputID(tx, scoid, txid) dbRemoveUnlockHash(tx, sco.UnlockHash, txid) } // Remove the file contract revision from the revision chain. dbRemoveFileContractRevision(tx, fcr.ParentID) } for _, sp := range txn.StorageProofs { dbRemoveStorageProof(tx, sp.ParentID) } for _, sfi := range txn.SiafundInputs { dbRemoveSiafundOutputID(tx, sfi.ParentID, txid) dbRemoveUnlockHash(tx, sfi.UnlockConditions.UnlockHash(), txid) dbRemoveUnlockHash(tx, sfi.ClaimUnlockHash, txid) } for k, sfo := range txn.SiafundOutputs { sfoid := txn.SiafundOutputID(uint64(k)) dbRemoveSiafundOutputID(tx, sfoid, txid) dbRemoveUnlockHash(tx, sfo.UnlockHash, txid) } } // remove the associated block facts dbRemoveBlockFacts(tx, bid) } // Update cumulative stats for applied blocks. for _, block := range cc.AppliedBlocks { bid := block.ID() tbid := types.TransactionID(bid) // special handling for genesis block if bid == types.GenesisID { dbAddGenesisBlock(tx) continue } blockheight++ dbAddBlockID(tx, bid, blockheight) dbAddTransactionID(tx, tbid, blockheight) // Miner payouts are a transaction target, exists := e.cs.ChildTarget(block.ParentID) if !exists { target = types.RootTarget } dbAddBlockTarget(tx, bid, target) // Catalog the new miner payouts. for j, payout := range block.MinerPayouts { scoid := block.MinerPayoutID(uint64(j)) dbAddSiacoinOutputID(tx, scoid, tbid) dbAddUnlockHash(tx, payout.UnlockHash, tbid) } // Update cumulative stats for applied transactions. for _, txn := range block.Transactions { // Add the transaction to the list of active transactions. txid := txn.ID() dbAddTransactionID(tx, txid, blockheight) for _, sci := range txn.SiacoinInputs { dbAddSiacoinOutputID(tx, sci.ParentID, txid) dbAddUnlockHash(tx, sci.UnlockConditions.UnlockHash(), txid) } for j, sco := range txn.SiacoinOutputs { scoid := txn.SiacoinOutputID(uint64(j)) dbAddSiacoinOutputID(tx, scoid, txid) dbAddUnlockHash(tx, sco.UnlockHash, txid) dbAddSiacoinOutput(tx, scoid, sco) } for k, fc := range txn.FileContracts { fcid := txn.FileContractID(uint64(k)) dbAddFileContractID(tx, fcid, txid) dbAddUnlockHash(tx, fc.UnlockHash, txid) dbAddFileContract(tx, fcid, fc) for l, sco := range fc.ValidProofOutputs { scoid := fcid.StorageProofOutputID(types.ProofValid, uint64(l)) dbAddSiacoinOutputID(tx, scoid, txid) dbAddUnlockHash(tx, sco.UnlockHash, txid) } for l, sco := range fc.MissedProofOutputs { scoid := fcid.StorageProofOutputID(types.ProofMissed, uint64(l)) dbAddSiacoinOutputID(tx, scoid, txid) dbAddUnlockHash(tx, sco.UnlockHash, txid) } } for _, fcr := range txn.FileContractRevisions { dbAddFileContractID(tx, fcr.ParentID, txid) dbAddUnlockHash(tx, fcr.UnlockConditions.UnlockHash(), txid) dbAddUnlockHash(tx, fcr.NewUnlockHash, txid) for l, sco := range fcr.NewValidProofOutputs { scoid := fcr.ParentID.StorageProofOutputID(types.ProofValid, uint64(l)) dbAddSiacoinOutputID(tx, scoid, txid) dbAddUnlockHash(tx, sco.UnlockHash, txid) } for l, sco := range fcr.NewMissedProofOutputs { scoid := fcr.ParentID.StorageProofOutputID(types.ProofMissed, uint64(l)) dbAddSiacoinOutputID(tx, scoid, txid) dbAddUnlockHash(tx, sco.UnlockHash, txid) } dbAddFileContractRevision(tx, fcr.ParentID, fcr) } for _, sp := range txn.StorageProofs { dbAddFileContractID(tx, sp.ParentID, txid) dbAddStorageProof(tx, sp.ParentID, sp) } for _, sfi := range txn.SiafundInputs { dbAddSiafundOutputID(tx, sfi.ParentID, txid) dbAddUnlockHash(tx, sfi.UnlockConditions.UnlockHash(), txid) dbAddUnlockHash(tx, sfi.ClaimUnlockHash, txid) } for k, sfo := range txn.SiafundOutputs { sfoid := txn.SiafundOutputID(uint64(k)) dbAddSiafundOutputID(tx, sfoid, txid) dbAddUnlockHash(tx, sfo.UnlockHash, txid) dbAddSiafundOutput(tx, sfoid, sfo) } } // calculate and add new block facts, if possible if tx.Bucket(bucketBlockFacts).Get(encoding.Marshal(block.ParentID)) != nil { facts := dbCalculateBlockFacts(tx, e.cs, block) dbAddBlockFacts(tx, facts) } } // Compute the changes in the active set. Note, because this is calculated // at the end instead of in a loop, the historic facts may contain // inaccuracies about the active set. This should not be a problem except // for large reorgs. // TODO: improve this currentBlock, exists := e.cs.BlockAtHeight(blockheight) if !exists { build.Critical("consensus is missing block", blockheight) } currentID := currentBlock.ID() var facts blockFacts err = dbGetAndDecode(bucketBlockFacts, currentID, &facts)(tx) if err == nil { for _, diff := range cc.FileContractDiffs { if diff.Direction == modules.DiffApply { facts.ActiveContractCount++ facts.ActiveContractCost = facts.ActiveContractCost.Add(diff.FileContract.Payout) facts.ActiveContractSize = facts.ActiveContractSize.Add(types.NewCurrency64(diff.FileContract.FileSize)) } else { facts.ActiveContractCount-- facts.ActiveContractCost = facts.ActiveContractCost.Sub(diff.FileContract.Payout) facts.ActiveContractSize = facts.ActiveContractSize.Sub(types.NewCurrency64(diff.FileContract.FileSize)) } } err = tx.Bucket(bucketBlockFacts).Put(encoding.Marshal(currentID), encoding.Marshal(facts)) if err != nil { return err } } // set final blockheight err = dbSetInternal(internalBlockHeight, blockheight)(tx) if err != nil { return err } // set change ID err = dbSetInternal(internalRecentChange, cc.ID)(tx) if err != nil { return err } return nil }) if err != nil { build.Critical("explorer update failed:", err) } } // helper functions func assertNil(err error) { if err != nil { panic(err) } } func mustPut(bucket *bolt.Bucket, key, val interface{}) { assertNil(bucket.Put(encoding.Marshal(key), encoding.Marshal(val))) } func mustPutSet(bucket *bolt.Bucket, key interface{}) { assertNil(bucket.Put(encoding.Marshal(key), nil)) } func mustDelete(bucket *bolt.Bucket, key interface{}) { assertNil(bucket.Delete(encoding.Marshal(key))) } // These functions panic on error. The panic will be caught by // ProcessConsensusChange. // Add/Remove block ID func dbAddBlockID(tx *bolt.Tx, id types.BlockID, height types.BlockHeight) { mustPut(tx.Bucket(bucketBlockIDs), id, height) } func dbRemoveBlockID(tx *bolt.Tx, id types.BlockID) { mustDelete(tx.Bucket(bucketBlockIDs), id) } // Add/Remove block facts func dbAddBlockFacts(tx *bolt.Tx, facts blockFacts) { mustPut(tx.Bucket(bucketBlockFacts), facts.BlockID, facts) } func dbRemoveBlockFacts(tx *bolt.Tx, id types.BlockID) { mustDelete(tx.Bucket(bucketBlockFacts), id) } // Add/Remove block target func dbAddBlockTarget(tx *bolt.Tx, id types.BlockID, target types.Target) { mustPut(tx.Bucket(bucketBlockTargets), id, target) } func dbRemoveBlockTarget(tx *bolt.Tx, id types.BlockID, target types.Target) { mustDelete(tx.Bucket(bucketBlockTargets), id) } // Add/Remove file contract func dbAddFileContract(tx *bolt.Tx, id types.FileContractID, fc types.FileContract) { history := fileContractHistory{Contract: fc} mustPut(tx.Bucket(bucketFileContractHistories), id, history) } func dbRemoveFileContract(tx *bolt.Tx, id types.FileContractID) { mustDelete(tx.Bucket(bucketFileContractHistories), id) } // Add/Remove txid from file contract ID bucket func dbAddFileContractID(tx *bolt.Tx, id types.FileContractID, txid types.TransactionID) { b, err := tx.Bucket(bucketFileContractIDs).CreateBucketIfNotExists(encoding.Marshal(id)) assertNil(err) mustPutSet(b, txid) } func dbRemoveFileContractID(tx *bolt.Tx, id types.FileContractID, txid types.TransactionID) { // TODO: delete bucket when it becomes empty mustDelete(tx.Bucket(bucketFileContractIDs).Bucket(encoding.Marshal(id)), txid) } func dbAddFileContractRevision(tx *bolt.Tx, fcid types.FileContractID, fcr types.FileContractRevision) { var history fileContractHistory assertNil(dbGetAndDecode(bucketFileContractHistories, fcid, &history)(tx)) history.Revisions = append(history.Revisions, fcr) mustPut(tx.Bucket(bucketFileContractHistories), fcid, history) } func dbRemoveFileContractRevision(tx *bolt.Tx, fcid types.FileContractID) { var history fileContractHistory assertNil(dbGetAndDecode(bucketFileContractHistories, fcid, &history)(tx)) // TODO: could be more rigorous history.Revisions = history.Revisions[:len(history.Revisions)-1] mustPut(tx.Bucket(bucketFileContractHistories), fcid, history) } // Add/Remove siacoin output func dbAddSiacoinOutput(tx *bolt.Tx, id types.SiacoinOutputID, output types.SiacoinOutput) { mustPut(tx.Bucket(bucketSiacoinOutputs), id, output) } func dbRemoveSiacoinOutput(tx *bolt.Tx, id types.SiacoinOutputID) { mustDelete(tx.Bucket(bucketSiacoinOutputs), id) } // Add/Remove txid from siacoin output ID bucket func dbAddSiacoinOutputID(tx *bolt.Tx, id types.SiacoinOutputID, txid types.TransactionID) { b, err := tx.Bucket(bucketSiacoinOutputIDs).CreateBucketIfNotExists(encoding.Marshal(id)) assertNil(err) mustPutSet(b, txid) } func dbRemoveSiacoinOutputID(tx *bolt.Tx, id types.SiacoinOutputID, txid types.TransactionID) { // TODO: delete bucket when it becomes empty mustDelete(tx.Bucket(bucketSiacoinOutputIDs).Bucket(encoding.Marshal(id)), txid) } // Add/Remove siafund output func dbAddSiafundOutput(tx *bolt.Tx, id types.SiafundOutputID, output types.SiafundOutput) { mustPut(tx.Bucket(bucketSiafundOutputs), id, output) } func dbRemoveSiafundOutput(tx *bolt.Tx, id types.SiafundOutputID) { mustDelete(tx.Bucket(bucketSiafundOutputs), id) } // Add/Remove txid from siafund output ID bucket func dbAddSiafundOutputID(tx *bolt.Tx, id types.SiafundOutputID, txid types.TransactionID) { b, err := tx.Bucket(bucketSiafundOutputIDs).CreateBucketIfNotExists(encoding.Marshal(id)) assertNil(err) mustPutSet(b, txid) } func dbRemoveSiafundOutputID(tx *bolt.Tx, id types.SiafundOutputID, txid types.TransactionID) { // TODO: delete bucket when it becomes empty mustDelete(tx.Bucket(bucketSiafundOutputIDs).Bucket(encoding.Marshal(id)), txid) } // Add/Remove storage proof func dbAddStorageProof(tx *bolt.Tx, fcid types.FileContractID, sp types.StorageProof) { var history fileContractHistory assertNil(dbGetAndDecode(bucketFileContractHistories, fcid, &history)(tx)) history.StorageProof = sp mustPut(tx.Bucket(bucketFileContractHistories), fcid, history) } func dbRemoveStorageProof(tx *bolt.Tx, fcid types.FileContractID) { dbAddStorageProof(tx, fcid, types.StorageProof{}) } // Add/Remove transaction ID func dbAddTransactionID(tx *bolt.Tx, id types.TransactionID, height types.BlockHeight) { mustPut(tx.Bucket(bucketTransactionIDs), id, height) } func dbRemoveTransactionID(tx *bolt.Tx, id types.TransactionID) { mustDelete(tx.Bucket(bucketTransactionIDs), id) } // Add/Remove txid from unlock hash bucket func dbAddUnlockHash(tx *bolt.Tx, uh types.UnlockHash, txid types.TransactionID) { b, err := tx.Bucket(bucketUnlockHashes).CreateBucketIfNotExists(encoding.Marshal(uh)) assertNil(err) mustPutSet(b, txid) } func dbRemoveUnlockHash(tx *bolt.Tx, uh types.UnlockHash, txid types.TransactionID) { // TODO: delete bucket when it becomes empty mustDelete(tx.Bucket(bucketUnlockHashes).Bucket(encoding.Marshal(uh)), txid) } func dbCalculateBlockFacts(tx *bolt.Tx, cs modules.ConsensusSet, block types.Block) blockFacts { // get the parent block facts var bf blockFacts err := dbGetAndDecode(bucketBlockFacts, block.ParentID, &bf)(tx) assertNil(err) // get target target, exists := cs.ChildTarget(block.ParentID) if !exists { panic(fmt.Sprint("ConsensusSet is missing target of known block", block.ParentID)) } // update fields bf.BlockID = block.ID() bf.Height++ bf.Difficulty = target.Difficulty() bf.Target = target bf.Timestamp = block.Timestamp bf.TotalCoins = types.CalculateNumSiacoins(bf.Height) // calculate maturity timestamp var maturityTimestamp types.Timestamp if bf.Height > types.MaturityDelay { oldBlock, exists := cs.BlockAtHeight(bf.Height - types.MaturityDelay) if !exists { panic(fmt.Sprint("ConsensusSet is missing block at height", bf.Height-types.MaturityDelay)) } maturityTimestamp = oldBlock.Timestamp } bf.MaturityTimestamp = maturityTimestamp // calculate hashrate by averaging last 'hashrateEstimationBlocks' blocks var estimatedHashrate types.Currency if bf.Height > hashrateEstimationBlocks { var totalDifficulty = bf.Target var oldestTimestamp types.Timestamp for i := types.BlockHeight(1); i < hashrateEstimationBlocks; i++ { b, exists := cs.BlockAtHeight(bf.Height - i) if !exists { panic(fmt.Sprint("ConsensusSet is missing block at height", bf.Height-hashrateEstimationBlocks)) } target, exists := cs.ChildTarget(b.ParentID) if !exists { panic(fmt.Sprint("ConsensusSet is missing target of known block", b.ParentID)) } totalDifficulty = totalDifficulty.AddDifficulties(target) oldestTimestamp = b.Timestamp } secondsPassed := bf.Timestamp - oldestTimestamp estimatedHashrate = totalDifficulty.Difficulty().Div64(uint64(secondsPassed)) } bf.EstimatedHashrate = estimatedHashrate bf.MinerPayoutCount += uint64(len(block.MinerPayouts)) bf.TransactionCount += uint64(len(block.Transactions)) for _, txn := range block.Transactions { bf.SiacoinInputCount += uint64(len(txn.SiacoinInputs)) bf.SiacoinOutputCount += uint64(len(txn.SiacoinOutputs)) bf.FileContractCount += uint64(len(txn.FileContracts)) bf.FileContractRevisionCount += uint64(len(txn.FileContractRevisions)) bf.StorageProofCount += uint64(len(txn.StorageProofs)) bf.SiafundInputCount += uint64(len(txn.SiafundInputs)) bf.SiafundOutputCount += uint64(len(txn.SiafundOutputs)) bf.MinerFeeCount += uint64(len(txn.MinerFees)) bf.ArbitraryDataCount += uint64(len(txn.ArbitraryData)) bf.TransactionSignatureCount += uint64(len(txn.TransactionSignatures)) for _, fc := range txn.FileContracts { bf.TotalContractCost = bf.TotalContractCost.Add(fc.Payout) bf.TotalContractSize = bf.TotalContractSize.Add(types.NewCurrency64(fc.FileSize)) } for _, fcr := range txn.FileContractRevisions { bf.TotalContractSize = bf.TotalContractSize.Add(types.NewCurrency64(fcr.NewFileSize)) bf.TotalRevisionVolume = bf.TotalRevisionVolume.Add(types.NewCurrency64(fcr.NewFileSize)) } } return bf } // Special handling for the genesis block. No other functions are called on it. func dbAddGenesisBlock(tx *bolt.Tx) { id := types.GenesisID dbAddBlockID(tx, id, 0) txid := types.GenesisBlock.Transactions[0].ID() dbAddTransactionID(tx, txid, 0) for i, sfo := range types.GenesisSiafundAllocation { sfoid := types.GenesisBlock.Transactions[0].SiafundOutputID(uint64(i)) dbAddSiafundOutputID(tx, sfoid, txid) dbAddUnlockHash(tx, sfo.UnlockHash, txid) dbAddSiafundOutput(tx, sfoid, sfo) } dbAddBlockFacts(tx, blockFacts{ BlockFacts: modules.BlockFacts{ BlockID: id, Height: 0, Difficulty: types.RootTarget.Difficulty(), Target: types.RootTarget, TotalCoins: types.CalculateCoinbase(0), TransactionCount: 1, SiafundOutputCount: uint64(len(types.GenesisSiafundAllocation)), }, Timestamp: types.GenesisBlock.Timestamp, }) } Sia-1.0.4/modules/explorer/update_test.go000066400000000000000000000137421302033070500204100ustar00rootroot00000000000000package explorer import ( "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) func (et *explorerTester) currentFacts() (facts modules.BlockFacts, exists bool) { var height types.BlockHeight err := et.explorer.db.View(dbGetInternal(internalBlockHeight, &height)) if err != nil { exists = false return } return et.explorer.BlockFacts(height) } // TestIntegrationExplorerFileContractMetrics checks that the siacoin // transfer volume metric is working correctly. func TestIntegrationExplorerFileContractMetrics(t *testing.T) { if testing.Short() { t.Skip() } et, err := createExplorerTester("TestIntegrationExporerFileContractMetrics") if err != nil { t.Fatal(err) } // Propel explorer tester past the hardfork height. for i := 0; i < 10; i++ { _, err = et.miner.AddBlock() if err != nil { t.Fatal(err) } } facts, ok := et.currentFacts() if !ok { t.Fatal("couldn't get current facts") } if !facts.ActiveContractCost.IsZero() { t.Error("fresh explorer has nonzero active contract cost") } if facts.ActiveContractCount != 0 { t.Error("active contract count should initialize to zero") } if !facts.ActiveContractSize.IsZero() { t.Error("active contract size should initialize to zero") } // Put a file contract into the chain, and check that the explorer // correctly does all of the counting. builder := et.wallet.StartTransaction() builder.FundSiacoins(types.NewCurrency64(5e9)) fcOutputs := []types.SiacoinOutput{{Value: types.NewCurrency64(4805e6)}} fc := types.FileContract{ FileSize: 5e3, WindowStart: et.cs.Height() + 2, WindowEnd: et.cs.Height() + 3, Payout: types.NewCurrency64(5e9), ValidProofOutputs: fcOutputs, MissedProofOutputs: fcOutputs, } _ = builder.AddFileContract(fc) txns, err := builder.Sign(true) if err != nil { t.Fatal(err) } err = et.tpool.AcceptTransactionSet(txns) if err != nil { t.Fatal(err) } _, err = et.miner.AddBlock() if err != nil { t.Fatal(err) } // Check that the stats have updated to represent the file contract. facts, ok = et.currentFacts() if !ok { t.Fatal("couldn't get current facts") } if !facts.ActiveContractCost.Equals64(5e9) { t.Error("active resources providing wrong file contract cost") } if facts.ActiveContractCount != 1 { t.Error("active contract count does not read correctly") } if !facts.ActiveContractSize.Equals64(5e3) { t.Error("active contract size is not correctly reported") } if !facts.TotalContractCost.Equals64(5e9) { t.Error("total cost is not tallied correctly") } if facts.FileContractCount != 1 { t.Error("total contract count is not accurate") } if !facts.TotalContractSize.Equals64(5e3) { t.Error("total contract size is not accurate") } // Put a second file into the explorer to check that multiple files are // handled well. builder = et.wallet.StartTransaction() builder.FundSiacoins(types.NewCurrency64(1e9)) fcOutputs = []types.SiacoinOutput{{Value: types.NewCurrency64(961e6)}} fc = types.FileContract{ FileSize: 15e3, WindowStart: et.cs.Height() + 2, WindowEnd: et.cs.Height() + 3, Payout: types.NewCurrency64(1e9), ValidProofOutputs: fcOutputs, MissedProofOutputs: fcOutputs, } _ = builder.AddFileContract(fc) txns, err = builder.Sign(true) if err != nil { t.Fatal(err) } err = et.tpool.AcceptTransactionSet(txns) if err != nil { t.Fatal(err) } _, err = et.miner.AddBlock() if err != nil { t.Fatal(err) } // Check that the stats have updated to represent the file contracts. facts, ok = et.currentFacts() if !ok { t.Fatal("couldn't get current facts") } if !facts.ActiveContractCost.Equals64(6e9) { t.Error("active resources providing wrong file contract cost") } if facts.ActiveContractCount != 2 { t.Error("active contract count does not read correctly") } if !facts.ActiveContractSize.Equals64(20e3) { t.Error("active contract size is not correctly reported") } if !facts.TotalContractCost.Equals64(6e9) { t.Error("total cost is not tallied correctly") } if facts.FileContractCount != 2 { t.Error("total contract count is not accurate") } if !facts.TotalContractSize.Equals64(20e3) { t.Error("total contract size is not accurate") } // Expire the first file contract but not the second. _, err = et.miner.AddBlock() if err != nil { t.Fatal(err) } // Check that the stats have updated to reflect the expired file contract. facts, ok = et.currentFacts() if !ok { t.Fatal("couldn't get current facts") } if !facts.ActiveContractCost.Equals64(1e9) { t.Error("active resources providing wrong file contract cost", facts.ActiveContractCost) } if facts.ActiveContractCount != 1 { t.Error("active contract count does not read correctly") } if !facts.ActiveContractSize.Equals64(15e3) { t.Error("active contract size is not correctly reported") } if !facts.TotalContractCost.Equals64(6e9) { t.Error("total cost is not tallied correctly") } if facts.FileContractCount != 2 { t.Error("total contract count is not accurate") } if !facts.TotalContractSize.Equals64(20e3) { t.Error("total contract size is not accurate") } // Reorg the block explorer to a blank state, see that all of the file // contract statistics got removed. // TODO: broken by new block facts model // err = et.reorgToBlank() // if err != nil { // t.Fatal(err) // } // facts, ok = et.currentFacts() // if !ok { // t.Fatal("couldn't get current facts") // } // if !facts.ActiveContractCost.IsZero() { // t.Error("post reorg active contract cost should be zero, got", facts.ActiveContractCost) // } // if facts.ActiveContractCount != 0 { // t.Error("post reorg active contract count should be zero, got", facts.ActiveContractCount) // } // if !facts.TotalContractCost.IsZero() { // t.Error("post reorg total contract cost should be zero, got", facts.TotalContractCost) // } // if facts.FileContractCount != 0 { // t.Error("post reorg file contract count should be zero, got", facts.FileContractCount) // } } Sia-1.0.4/modules/gateway.go000066400000000000000000000133661302033070500156720ustar00rootroot00000000000000package modules import ( "net" "github.com/NebulousLabs/Sia/build" ) const ( // GatewayDir is the name of the directory used to store the gateway's // persistent data. GatewayDir = "gateway" ) var ( // BootstrapPeers is a list of peers that can be used to find other peers - // when a client first connects to the network, the only options for // finding peers are either manual entry of peers or to use a hardcoded // bootstrap point. While the bootstrap point could be a central service, // it can also be a list of peers that are known to be stable. We have // chosen to hardcode known-stable peers. BootstrapPeers = func() []NetAddress { switch build.Release { case "dev": return nil case "standard": return []NetAddress{ "101.200.214.115:9981", "104.223.98.174:9981", "109.172.42.157:9981", "109.206.33.225:9981", "109.71.42.163:9981", "109.71.42.164:9981", "113.98.98.164:9981", "115.187.229.102:9981", "120.25.198.251:9981", "138.201.13.159:9981", "139.162.152.204:9981", "141.105.11.33:9981", "142.4.209.72:9981", "148.251.221.163:9981", "158.69.120.71:9981", "162.210.249.170:9981", "162.222.23.93:9981", "176.9.59.110:9981", "176.9.72.2:9981", "180.167.17.236:9981", "18.239.0.53:9981", "183.86.218.232:9981", "188.166.61.155:9981", "188.166.61.157:9981", "188.166.61.158:9981", "188.166.61.159:9981", "188.166.61.163:9981", "188.61.177.92:9981", "190.10.8.173:9981", "193.198.102.34:9981", "194.135.90.38:9981", "195.154.243.233:9981", "202.63.55.79:9981", "210.14.155.90:9981", "213.251.158.199:9981", "217.65.8.75:9981", "222.187.224.89:9981", "222.187.224.93:9981", "23.239.14.98:9971", "23.239.14.98:9981", "23.239.14.98:9981", "24.91.0.62:9981", "31.178.227.21:9981", "37.139.1.78:9981", "37.139.28.207:9981", "43.227.113.131:9981", "45.79.132.35:9981", "45.79.159.167:9981", "46.105.118.15:9981", "52.205.238.6:9981", "62.107.201.170:9981", "62.210.207.79:9981", "62.212.77.12:9981", "64.31.31.106:9981", "68.55.10.144:9981", "73.73.50.191:33721", "76.164.234.13:9981", "78.119.218.13:9981", "79.172.204.10:9981", "79.51.183.132:9981", "80.234.37.94:9981", "82.196.11.170:9981", "82.196.5.50:9981", "82.220.99.82:9981", "83.76.19.197:10981", "85.255.197.69:9981", "87.236.27.155:12487", "87.98.216.46:9981", "88.109.8.173:9981", "91.121.183.171:9981", "95.211.203.138:9981", "95.85.14.54:9981", "95.85.15.69:9981", "95.85.15.71:9981", } case "testing": return nil default: panic("unrecognized build.Release constant in BootstrapPeers") } }() ) type ( // Peer contains all the info necessary to Broadcast to a peer. Peer struct { Inbound bool `json:"inbound"` Local bool `json:"local"` NetAddress NetAddress `json:"netaddress"` Version string `json:"version"` } // A PeerConn is the connection type used when communicating with peers during // an RPC. It is identical to a net.Conn with the additional RPCAddr method. // This method acts as an identifier for peers and is the address that the // peer can be dialed on. It is also the address that should be used when // calling an RPC on the peer. PeerConn interface { net.Conn RPCAddr() NetAddress } // RPCFunc is the type signature of functions that handle RPCs. It is used for // both the caller and the callee. RPCFuncs may perform locking. RPCFuncs may // close the connection early, and it is recommended that they do so to avoid // keeping the connection open after all necessary I/O has been performed. RPCFunc func(PeerConn) error // A Gateway facilitates the interactions between the local node and remote // nodes (peers). It relays incoming blocks and transactions to local modules, // and broadcasts outgoing blocks and transactions to peers. In a broad sense, // it is responsible for ensuring that the local consensus set is consistent // with the "network" consensus set. Gateway interface { // Connect establishes a persistent connection to a peer. Connect(NetAddress) error // Disconnect terminates a connection to a peer. Disconnect(NetAddress) error // Address returns the Gateway's address. Address() NetAddress // Peers returns the addresses that the Gateway is currently connected to. Peers() []Peer // RegisterRPC registers a function to handle incoming connections that // supply the given RPC ID. RegisterRPC(string, RPCFunc) // UnregisterRPC unregisters an RPC and removes all references to the RPCFunc // supplied in the corresponding RegisterRPC call. References to RPCFuncs // registered with RegisterConnectCall are not removed and should be removed // with UnregisterConnectCall. If the RPC does not exist no action is taken. UnregisterRPC(string) // RegisterConnectCall registers an RPC name and function to be called // upon connecting to a peer. RegisterConnectCall(string, RPCFunc) // UnregisterConnectCall unregisters an RPC and removes all references to the // RPCFunc supplied in the corresponding RegisterConnectCall call. References // to RPCFuncs registered with RegisterRPC are not removed and should be // removed with UnregisterRPC. If the RPC does not exist no action is taken. UnregisterConnectCall(string) // RPC calls an RPC on the given address. RPC cannot be called on an // address that the Gateway is not connected to. RPC(NetAddress, string, RPCFunc) error // Broadcast transmits obj, prefaced by the RPC name, to all of the // given peers in parallel. Broadcast(name string, obj interface{}, peers []Peer) // Close safely stops the Gateway's listener process. Close() error } ) Sia-1.0.4/modules/gateway/000077500000000000000000000000001302033070500153325ustar00rootroot00000000000000Sia-1.0.4/modules/gateway/conn.go000066400000000000000000000016311302033070500166170ustar00rootroot00000000000000package gateway import ( "net" "time" "github.com/NebulousLabs/Sia/modules" ) // peerConn is a simple type that implements the modules.PeerConn interface. type peerConn struct { net.Conn dialbackAddr modules.NetAddress } // RPCAddr implements the RPCAddr method of the modules.PeerConn interface. It // is the address that identifies a peer. func (pc peerConn) RPCAddr() modules.NetAddress { return pc.dialbackAddr } // dial will dial the input address and return a connection. dial appropriately // handles things like clean shutdown, fast shutdown, and chooses the correct // communication protocol. func (g *Gateway) dial(addr modules.NetAddress) (net.Conn, error) { dialer := &net.Dialer{ Cancel: g.threads.StopChan(), Timeout: dialTimeout, } conn, err := dialer.Dial("tcp", string(addr)) if err != nil { return nil, err } conn.SetDeadline(time.Now().Add(connStdDeadline)) return conn, nil } Sia-1.0.4/modules/gateway/consts.go000066400000000000000000000167511302033070500172040ustar00rootroot00000000000000package gateway import ( "time" "github.com/NebulousLabs/Sia/build" ) const ( // handshakeUpgradeVersion is the version where the gateway handshake RPC // was altered to include adiitional information transfer. handshakeUpgradeVersion = "1.0.0" // maxLocalOutbound is currently set to 3, meaning the gateway will not // consider a local node to be an outbound peer if the gateway already has // 3 outbound peers. Three is currently needed to handle situations where // the gateway is at high risk of connecting to itself (such as a low // number of total peers, especially such as in a testing environment). // Once the gateway has a proper way to figure out that it's trying to // connect to itself, this number can be reduced. maxLocalOutboundPeers = 3 // minAcceptableVersion is the version below which the gateway will refuse to // connect to peers and reject connection attempts. // // Reject peers < v0.4.0 as the previous version is v0.3.3 which is // pre-hardfork. minAcceptableVersion = "0.4.0" ) var ( // healthyNodeListLen defines the number of nodes that the gateway must // have in the node list before it will stop asking peers for more nodes. healthyNodeListLen = func() int { switch build.Release { case "dev": return 30 case "standard": return 200 case "testing": return 15 default: panic("unrecognized build.Release in healthyNodeListLen") } }() // maxSharedNodes defines the number of nodes that will be shared between // peers when they are expanding their node lists. maxSharedNodes = func() uint64 { switch build.Release { case "dev": return 5 case "standard": return 10 case "testing": return 3 default: panic("unrecognized build.Release in healthyNodeListLen") } }() // nodePurgeDelay defines the amount of time that is waited between each // iteration of the node purge loop. nodePurgeDelay = func() time.Duration { switch build.Release { case "dev": return 20 * time.Second case "standard": return 10 * time.Minute case "testing": return 500 * time.Millisecond default: panic("unrecognized build.Release in nodePurgeDelay") } }() // nodeListDelay defines the amount of time that is waited between each // iteration of the node list loop. nodeListDelay = func() time.Duration { switch build.Release { case "dev": return 3 * time.Second case "standard": return 5 * time.Second case "testing": return 500 * time.Millisecond default: panic("unrecognized build.Release in nodePurgeDelay") } }() // pruneNodeListLen defines the number of nodes that the gateway must have // to be pruning nodes from the node list. pruneNodeListLen = func() int { switch build.Release { case "dev": return 15 case "standard": return 50 case "testing": return 10 default: panic("unrecognized build.Release in pruneNodeListLen") } }() ) var ( // The gateway will sleep this long between incoming connections. For // attack reasons, the acceptInterval should be longer than the // nodeListDelay. Right at startup, a node is vulnerable to being flooded // by Sybil attackers. The node's best defense is to wait until it has // filled out its nodelist somewhat from the bootstrap nodes. An attacker // needs to completely dominate the nodelist and the peerlist to be // successful, so just a few honest nodes from requests to the bootstraps // should be enough to fend from most attacks. acceptInterval = func() time.Duration { switch build.Release { case "dev": return 3 * time.Second case "standard": return 6 * time.Second case "testing": return 100 * time.Millisecond default: panic("unrecognized build.Release in acceptInterval") } }() // acquiringPeersDelay defines the amount of time that is waited between // iterations of the peer acquisition loop if the gateway is actively // forming new connections with peers. acquiringPeersDelay = func() time.Duration { switch build.Release { case "dev": return 3 * time.Second case "standard": return 5 * time.Second case "testing": return 500 * time.Millisecond default: panic("unrecognized build.Release in wellConnectedDelay") } }() // fullyConnectedThreshold defines the number of peers that the gateway can // have before it stops accepting inbound connections. fullyConnectedThreshold = func() int { switch build.Release { case "dev": return 20 case "standard": return 128 case "testing": return 10 default: panic("unrecognized build.Release in fullyConnectedThreshold") } }() // maxConcurrentOutboundPeerRequests defines the maximum number of peer // connections that the gateway will try to form concurrently. maxConcurrentOutboundPeerRequests = func() int { switch build.Release { case "dev": return 2 case "standard": return 3 case "testing": return 2 default: panic("unrecognized build.Release in maxConcurrentOutboundPeerRequests") } }() // noNodesDelay defines the amount of time that is waited between // iterations of the peer acquisition loop if the gateway does not have any // nodes in the nodelist. noNodesDelay = func() time.Duration { switch build.Release { case "dev": return 10 * time.Second case "standard": return 20 * time.Second case "testing": return 3 * time.Second default: panic("unrecognized build.Release in noNodesDelay") } }() // unwawntedLocalPeerDelay defines the amount of time that is waited // between iterations of the permanentPeerManager if the gateway has at // least a few outbound peers, but is not well connected, and the recently // selected peer was a local peer. The wait is mostly to prevent the // gateway from hogging the CPU in the event that all peers are local // peers. unwantedLocalPeerDelay = func() time.Duration { switch build.Release { case "dev": return 1 * time.Second case "standard": return 2 * time.Second case "testing": return 100 * time.Millisecond default: panic("unrecognized build.Release in unwawntedLocalPeerDelay") } }() // wellConnectedDelay defines the amount of time that is waited between // iterations of the peer acquisition loop if the gateway is well // connected. wellConnectedDelay = func() time.Duration { switch build.Release { case "dev": return 1 * time.Minute case "standard": return 5 * time.Minute case "testing": return 3 * time.Second default: panic("unrecognized build.Release in wellConnectedDelay") } }() // wellConnectedThreshold is the number of outbound connections at which // the gateway will not attempt to make new outbound connections. wellConnectedThreshold = func() int { switch build.Release { case "dev": return 5 case "standard": return 8 case "testing": return 4 default: panic("unrecognized build.Release in wellConnectedThreshold") } }() ) var ( // connStdDeadline defines the standard deadline that should be used for // all temporary connections to the gateway. connStdDeadline = func() time.Duration { switch build.Release { case "dev": return 2 * time.Minute case "standard": return 5 * time.Minute case "testing": return 30 * time.Second default: panic("unrecognized build.Release in connStdDeadline") } }() // the gateway will abort a connection attempt after this long dialTimeout = func() time.Duration { switch build.Release { case "dev": return 20 * time.Second case "standard": return 2 * time.Minute case "testing": return 500 * time.Millisecond default: panic("unrecognized build.Release in dialTimeout") } }() ) Sia-1.0.4/modules/gateway/gateway.go000066400000000000000000000347451302033070500173370ustar00rootroot00000000000000// Package gateway connects a Sia node to the Sia flood network. The flood // network is used to propagate blocks and transactions. The gateway is the // primary avenue that a node uses to hear about transactions and blocks, and // is the primary avenue used to tell the network about blocks that you have // mined or about transactions that you have created. package gateway import ( "time" ) // For the user to be securely connected to the network, the user must be // connected to at least one node which will send them all of the blocks. An // attacker can trick the user into thinking that a different blockchain is the // full blockchain if the user is not connected to any nodes who are seeing + // broadcasting the real chain (and instead is connected only to attacker nodes // or to nodes that are not broadcasting). This situation is called an eclipse // attack. // // Connecting to a large number of nodes increases the resiliancy of the // network, but also puts a networking burden on the nodes and can slow down // block propagation or increase orphan rates. The gateway's job is to keep the // network efficient while also protecting the user against attacks. // // The gateway keeps a list of nodes that it knows about. It uses this list to // form connections with other nodes, and then uses those connections to // participate in the flood network. The primary vector for an attacker to // achieve an eclipse attack is node list domination. If a gateway's nodelist // is heavily dominated by attacking nodes, then when the gateway chooses to // make random connections the gateway is at risk of selecting only attacker // nodes. // // The gateway defends itself from these attacks by minimizing the amount of // control that an attacker has over the node list and peer list. The first // major defense is that the gateway maintains 8 'outbound' relationships, // which means that the gateway created those relationships instead of an // attacker. If a node forms a connection to you, that node is called // 'inbound', and because it may be an attacker node, it is not trusted. // Outbound nodes can also be attacker nodes, but they are less likely to be // attacker nodes because you chose them, instead of them choosing you. // // If the gateway forms too many connections, the gateway will allow incoming // connections by kicking an existing peer. But, to limit the amount of control // that an attacker may have, only inbound peers are selected to be kicked. // Furthermore, to increase the difficulty of attack, if a new inbound // connection shares the same IP address as an existing connection, the shared // connection is the connection that gets dropped (unless that connection is a // local or outbound connection). // // Nodes are added to a peerlist in two methods. The first method is that a // gateway will ask its outbound peers for a list of nodes. If the node list is // below a certain size (see consts.go), the gateway will repeatedly ask // outbound peers to expand the list. Nodes are also added to the nodelist // after they successfully form a connection with the gateway. To limit the // attacker's ability to add nodes to the nodelist, connections are // ratelimited. An attacker with lots of IP addresses still has the ability to // fill up the nodelist, however getting 90% dominance of the nodelist requries // forming thousands of connections, which will take hours or days. By that // time, the attacked node should already have its set of outbound peers, // limiting the amount of damage that the attacker can do. // // To limit DNS-based tomfoolry, nodes are only added to the nodelist if their // connection information takes the form of an IP address. // // Some research has been done on Bitcoin's flood networks. The more relevant // research has been listed below. The papers listed first are more relevant. // Eclipse Attacks on Bitcoin's Peer-to-Peer Network (Heilman, Kendler, Zohar, Goldberg) // Stubborn Mining: Generalizing Selfish Mining and Combining with an Eclipse Attack (Nayak, Kumar, Miller, Shi) // An Overview of BGP Hijacking (https://www.bishopfox.com/blog/2015/08/an-overview-of-bgp-hijacking/) // TODO: Currently the gateway does not do much in terms of bucketing. The // gateway should make sure that it has outbound peers from a wide range of IP // addresses, and when kicking inbound peers it shouldn't just favor kicking // peers of the same IP address, it should favor kicking peers of the same ip // address range. // // TODO: Currently the gateway does not save a list of its outbound // connections. When it restarts, it will have a full nodelist (which may be // primarily attacker nodes) and it will be connecting primarily to nodes in // the nodelist. Instead, it should start by trying to connect to peers that // have previously been outbound peers, as it is less likely that those have // been manipulated. // // TODO: When peers connect to eachother, and when they add nodes to the node // list, there is no verification that the peers are running on the same Sia // network, something that will be problematic if we set up a large testnet. // It's already problematic, and currently only solved by using a different set // of bootstrap nodes. If there is any cross-polination (which an attacker // could do pretty easily), the gateways will not clean up over time, which // will degrade the quality of the flood network as the two networks will // continously flood eachother with irrelevant information. Additionally, there // is no public key exhcange, so communications cannot be effectively encrypted // or authenticated. The nodes must have some way to share keys. // // TODO: There is currently no way for a user to choose the bootstrap nodes // that the gateway uses, short of modifying the source code. A method for // manipulating the bootstrap node list (before startup) is desired. // // TODO: Gateway hostname discovery currently has significant centralization, // namely the fallback is a single third-party website that can easily form any // response it wants. Instead, multiple TLS-protected third party websites // should be used, and the plurality answer should be accepted as the true // hostname. // // TODO: The gateway currently does hostname discovery in a non-blocking way, // which means that the first few peers that it connects to may not get the // correct hostname. This means that you may give the remote peer the wrong // hostname, which means they will not be able to dial you back, which means // they will not add you to their node list. // // TODO: The gateway should encrypt and authenticate all communications. Though // the gateway participates in a flood network, practical attacks have been // demonstrated which have been able to confuse nodes by manipulating messages // from their peers. Encryption + authentication would have made the attack // more difficult. // // TODO: The gateway does an unofficial ping in two places within nodes.go. // These unofficial pings can be found by searching for the string "0.0.0". To // perform the unofficial ping, the gateway connects to a peer and writes its // version as 0.0.0. When the connection completes and the handshake passes, // the gateway writes its version as '0.0.0', causing the other node to reject // the connection and disconnect. The gateway then can tell that the node on // the other end of the line is a Sia gateway, without having to add that node // as a peer. A better solution to this hack is to just add an RPC for the ping // function. Rollout will of course have to be gradual, as new nodes and old // nodes need to be able to successfully ping eachother until a sufficient // portion of the network has upgraded to the new code. // // TODO: The gateway, when connecting to a peer, currently has no way of // recognizing whether that peer is itself or not. Because of this, the gateway // does occasionally try to connect to itself, especially if it has a really // small node list. This can cause problems, especially during testing, if the // gateway is intentionally limiting the number of local outbount peers that it // has. For that reason, the 'permanentPeerManager' will allow the gateway to // add local peers as outbound peers all the way up to 3 total connections. The // gateway needs a proper way to recognize if it is trying to connect itself // (sending a nonce when performing the version handshake, for example). Once // that has been implemented, maxLocalOutobundPeers can be reduced to 2 or 1 // (probably 2). import ( "errors" "fmt" "net" "os" "path/filepath" "sync" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" siasync "github.com/NebulousLabs/Sia/sync" ) var ( errNoPeers = errors.New("no peers") errUnreachable = errors.New("peer did not respond to ping") ) // Gateway implements the modules.Gateway interface. type Gateway struct { listener net.Listener myAddr modules.NetAddress port string // handlers are the RPCs that the Gateway can handle. // // initRPCs are the RPCs that the Gateway calls upon connecting to a peer. handlers map[rpcID]modules.RPCFunc initRPCs map[string]modules.RPCFunc // nodes is the set of all known nodes (i.e. potential peers). // // peers are the nodes that the gateway is currently connected to. // // peerTG is a special thread group for tracking peer connections, and will // block shutdown until all peer connections have been closed out. The peer // connections are put in a separate TG because of their unique // requirements - they have the potential to live for the lifetime of the // program, but also the potential to close early. Calling threads.OnStop // for each peer could create a huge backlog of functions that do nothing // (because most of the peers disconnected prior to shutdown). And they // can't call threads.Add because they are potentially very long running // and would block any threads.Flush() calls. So a second threadgroup is // added which handles clean-shutdown for the peers, without blocking // threads.Flush() calls. nodes map[modules.NetAddress]struct{} peers map[modules.NetAddress]*peer peerTG siasync.ThreadGroup // Utilities. log *persist.Logger mu sync.RWMutex persistDir string threads siasync.ThreadGroup } // managedSleep will sleep for the given period of time. If the full time // elapses, 'false' is returned. If the sleep is interrupted for shutdown, // 'true' is returned. func (g *Gateway) managedSleep(t time.Duration) (completed bool) { select { case <-time.After(t): return true case <-g.threads.StopChan(): return false } } // Address returns the NetAddress of the Gateway. func (g *Gateway) Address() modules.NetAddress { g.mu.RLock() defer g.mu.RUnlock() return g.myAddr } // Close saves the state of the Gateway and stops its listener process. func (g *Gateway) Close() error { if err := g.threads.Stop(); err != nil { return err } g.mu.RLock() defer g.mu.RUnlock() return g.saveSync() } // New returns an initialized Gateway. func New(addr string, bootstrap bool, persistDir string) (*Gateway, error) { // Create the directory if it doesn't exist. err := os.MkdirAll(persistDir, 0700) if err != nil { return nil, err } g := &Gateway{ handlers: make(map[rpcID]modules.RPCFunc), initRPCs: make(map[string]modules.RPCFunc), peers: make(map[modules.NetAddress]*peer), nodes: make(map[modules.NetAddress]struct{}), persistDir: persistDir, } // Create the logger. g.log, err = persist.NewFileLogger(filepath.Join(g.persistDir, logFile)) if err != nil { return nil, err } // Establish the closing of the logger. g.threads.AfterStop(func() { if err := g.log.Close(); err != nil { // The logger may or may not be working here, so use a println // instead. fmt.Println("Failed to close the gateway logger:", err) } }) // Establish that the peerTG must complete shutdown before the primary // thread group completes shutdown. g.threads.OnStop(func() { err = g.peerTG.Stop() if err != nil { g.log.Println("ERROR: peerTG experienced errors while shutting down:", err) } }) // Register RPCs. g.RegisterRPC("ShareNodes", g.shareNodes) g.RegisterConnectCall("ShareNodes", g.requestNodes) // Establish the de-registration of the RPCs. g.threads.OnStop(func() { g.UnregisterRPC("ShareNodes") g.UnregisterConnectCall("ShareNodes") }) // Load the old node list. If it doesn't exist, no problem, but if it does, // we want to know about any errors preventing us from loading it. if loadErr := g.load(); loadErr != nil && !os.IsNotExist(loadErr) { return nil, loadErr } // Add the bootstrap peers to the node list. if bootstrap { for _, addr := range modules.BootstrapPeers { err := g.addNode(addr) if err != nil && err != errNodeExists { g.log.Printf("WARN: failed to add the bootstrap node '%v': %v", addr, err) } } } // Create the listener which will listen for new connections from peers. permanentListenClosedChan := make(chan struct{}) g.listener, err = net.Listen("tcp", addr) if err != nil { return nil, err } // Automatically close the listener when g.threads.Stop() is called. g.threads.OnStop(func() { err := g.listener.Close() if err != nil { g.log.Println("WARN: closing the listener failed:", err) } <-permanentListenClosedChan }) // Set the address and port of the gateway. _, g.port, err = net.SplitHostPort(g.listener.Addr().String()) if err != nil { return nil, err } // Set myAddr equal to the address returned by the listener. It will be // overwritten by threadedLearnHostname later on. g.myAddr = modules.NetAddress(g.listener.Addr().String()) // Spawn the peer connection listener. go g.permanentListen(permanentListenClosedChan) // Spawn the peer manager and provide tools for ensuring clean shutdown. peerManagerClosedChan := make(chan struct{}) g.threads.OnStop(func() { <-peerManagerClosedChan }) go g.permanentPeerManager(peerManagerClosedChan) // Spawn the node manager and provide tools for ensuring clean shudown. nodeManagerClosedChan := make(chan struct{}) g.threads.OnStop(func() { <-nodeManagerClosedChan }) go g.permanentNodeManager(nodeManagerClosedChan) // Spawn the node purger and provide tools for ensuring clean shutdown. nodePurgerClosedChan := make(chan struct{}) g.threads.OnStop(func() { <-nodePurgerClosedChan }) go g.permanentNodePurger(nodePurgerClosedChan) // Spawn threads to take care of port forwarding and hostname discovery. go g.threadedForwardPort(g.port) go g.threadedLearnHostname() g.log.Println("INFO: gateway created, started logging") return g, nil } // enforce that Gateway satisfies the modules.Gateway interface var _ modules.Gateway = (*Gateway)(nil) Sia-1.0.4/modules/gateway/gateway_test.go000066400000000000000000000116141302033070500203640ustar00rootroot00000000000000package gateway import ( "io/ioutil" "net" "os" "path/filepath" "sync" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" siasync "github.com/NebulousLabs/Sia/sync" ) // newTestingGateway returns a gateway read to use in a testing environment. func newTestingGateway(name string, t *testing.T) *Gateway { if testing.Short() { panic("newTestingGateway called during short test") } g, err := New("localhost:0", false, build.TempDir("gateway", name)) if err != nil { panic(err) } return g } // TestExportedMethodsErrAfterClose tests that exported methods like Close and // Connect error with siasync.ErrStopped after the gateway has been closed. func TestExportedMethodsErrAfterClose(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestCloseErrsSecondTime", t) if err := g.Close(); err != nil { t.Fatal(err) } if err := g.Close(); err != siasync.ErrStopped { t.Fatalf("expected %q, got %q", siasync.ErrStopped, err) } if err := g.Connect("localhost:1234"); err != siasync.ErrStopped { t.Fatalf("expected %q, got %q", siasync.ErrStopped, err) } } // TestAddress tests that Gateway.Address returns the address of its listener. // Also tests that the address is not unspecified and is a loopback address. // The address must be a loopback address for testing. func TestAddress(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestAddress", t) defer g.Close() if g.Address() != g.myAddr { t.Fatal("Address does not return g.myAddr") } if g.Address() != modules.NetAddress(g.listener.Addr().String()) { t.Fatalf("wrong address: expected %v, got %v", g.listener.Addr(), g.Address()) } host := modules.NetAddress(g.listener.Addr().String()).Host() ip := net.ParseIP(host) if ip == nil { t.Fatal("address is not an IP address") } if ip.IsUnspecified() { t.Fatal("expected a non-unspecified address") } if !ip.IsLoopback() { t.Fatal("expected a loopback address") } } // TestPeers checks that two gateways are able to connect to each other. func TestPeers(t *testing.T) { if testing.Short() { t.SkipNow() } g1 := newTestingGateway("TestRPC1", t) defer g1.Close() g2 := newTestingGateway("TestRPC2", t) defer g2.Close() err := g1.Connect(g2.Address()) if err != nil { t.Fatal("failed to connect:", err) } peers := g1.Peers() if len(peers) != 1 || peers[0].NetAddress != g2.Address() { t.Fatal("g1 has bad peer list:", peers) } err = g1.Disconnect(g2.Address()) if err != nil { t.Fatal("failed to disconnect:", err) } peers = g1.Peers() if len(peers) != 0 { t.Fatal("g1 has peers after disconnect:", peers) } } // TestNew checks that a call to New is effective. func TestNew(t *testing.T) { if testing.Short() { t.SkipNow() } if _, err := New("", false, ""); err == nil { t.Fatal("expecting persistDir error, got nil") } if _, err := New("localhost:0", false, ""); err == nil { t.Fatal("expecting persistDir error, got nil") } if g, err := New("foo", false, build.TempDir("gateway", "TestNew1")); err == nil { t.Fatal("expecting listener error, got nil", g.myAddr) } // create corrupted nodes.json dir := build.TempDir("gateway", "TestNew2") os.MkdirAll(dir, 0700) err := ioutil.WriteFile(filepath.Join(dir, "nodes.json"), []byte{1, 2, 3}, 0660) if err != nil { t.Fatal("couldn't create corrupted file:", err) } if _, err := New("localhost:0", false, dir); err == nil { t.Fatal("expected load error, got nil") } } // TestClose creates and closes a gateway. func TestClose(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() g := newTestingGateway("TestClose", t) err := g.Close() if err != nil { t.Fatal(err) } } // TestParallelClose spins up 3 gateways, connects them all, and then closes // them in parallel. The goal of this test is to make it more vulnerable to any // potential nondeterministic failures. func TestParallelClose(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // Spin up three gateways in parallel. var g1, g2, g3 *Gateway var wg sync.WaitGroup wg.Add(3) go func() { g1 = newTestingGateway("TestParallelClose - 1", t) wg.Done() }() go func() { g2 = newTestingGateway("TestParallelClose - 2", t) wg.Done() }() go func() { g3 = newTestingGateway("TestParallelClose - 3", t) wg.Done() }() wg.Wait() // Connect g1 to g2, g2 to g3. They may connect to eachother further. wg.Add(2) go func() { err := g1.Connect(g2.myAddr) if err != nil { panic(err) } wg.Done() }() go func() { err := g2.Connect(g3.myAddr) if err != nil { panic(err) } wg.Done() }() wg.Wait() // Close all three gateways in parallel. wg.Add(3) go func() { err := g1.Close() if err != nil { panic(err) } wg.Done() }() go func() { err := g2.Close() if err != nil { panic(err) } wg.Done() }() go func() { err := g3.Close() if err != nil { panic(err) } wg.Done() }() wg.Wait() } Sia-1.0.4/modules/gateway/nodes.go000066400000000000000000000226101302033070500167720ustar00rootroot00000000000000package gateway import ( "errors" "net" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" ) var ( errNodeExists = errors.New("node already added") errNoNodes = errors.New("no nodes in the node list") errOurAddress = errors.New("can't add our own address") ) // addNode adds an address to the set of nodes on the network. func (g *Gateway) addNode(addr modules.NetAddress) error { if addr == g.myAddr { return errOurAddress } else if _, exists := g.nodes[addr]; exists { return errNodeExists } else if addr.IsStdValid() != nil { return errors.New("address is not valid: " + string(addr)) } else if net.ParseIP(addr.Host()) == nil { return errors.New("address must be an IP address: " + string(addr)) } g.nodes[addr] = struct{}{} return nil } // managedAddUntrustedNode adds an address to the set of nodes on the network, but // first verifies that there is a reachable node at the provided address. func (g *Gateway) managedAddUntrustedNode(addr modules.NetAddress) error { // Performing the ping during testing does not work. if build.Release == "testing" { g.mu.Lock() defer g.mu.Unlock() return g.addNode(addr) } // Ping the untrusted node to see whether or not there's acutally a // reachable node at the provided address. conn, err := g.dial(addr) if err != nil { return err } // If connection succeeds, supply an unacceptable version so that we // will not be added as a peer. // // NOTE: this is a somewhat clunky way of specifying that you didn't // actually want a connection. encoding.WriteObject(conn, "0.0.0") var reject string err = encoding.ReadObject(conn, &reject, build.MaxEncodedVersionLength) if err != nil { g.log.Debugln("ERROR: version handshake ping terminated unexpectedly:", err) } if reject != "reject" { g.log.Debugln("WARN: peer does not seem to have correctly rejected our ping:", reject) } conn.Close() g.mu.Lock() defer g.mu.Unlock() err = g.addNode(addr) if err != nil { return err } return g.save() } // removeNode will remove a node from the gateway. func (g *Gateway) removeNode(addr modules.NetAddress) error { if _, exists := g.nodes[addr]; !exists { return errors.New("no record of that node") } delete(g.nodes, addr) return nil } // randomNode returns a random node from the gateway. An error can be returned // if there are no nodes in the node list. func (g *Gateway) randomNode() (modules.NetAddress, error) { if len(g.nodes) == 0 { return "", errNoPeers } // Select a random peer. Note that the algorithm below is roughly linear in // the number of nodes known by the gateway, and this number can approach // every node on the network. If the network gets large, this algorithm // will either need to be refactored, or more likely a cap on the size of // g.nodes will need to be added. r, err := crypto.RandIntn(len(g.nodes)) if err != nil { return "", err } for node := range g.nodes { if r <= 0 { return node, nil } r-- } return "", errNoPeers } // shareNodes is the receiving end of the ShareNodes RPC. It writes up to 10 // randomly selected nodes to the caller. func (g *Gateway) shareNodes(conn modules.PeerConn) error { conn.SetDeadline(time.Now().Add(connStdDeadline)) // Assemble a list of nodes to send to the peer. var nodes []modules.NetAddress func() { g.mu.RLock() defer g.mu.RUnlock() // Create a random permutation of nodes from the gateway to iterate // through. gnodes := make([]modules.NetAddress, 0, len(g.nodes)) for node := range g.nodes { gnodes = append(gnodes, node) } perm, err := crypto.Perm(len(g.nodes)) if err != nil { g.log.Severe("Unable to get random permutation for sharing nodes") } // Iterate through the random permutation of nodes and select the // desirable ones. remoteNA := modules.NetAddress(conn.RemoteAddr().String()) for _, i := range perm { // Don't share local peers with remote peers. That means that if 'node' // is loopback, it will only be shared if the remote peer is also // loopback. And if 'node' is private, it will only be shared if the // remote peer is either the loopback or is also private. node := gnodes[i] if node.IsLoopback() && !remoteNA.IsLoopback() { continue } if node.IsLocal() && !remoteNA.IsLocal() { continue } nodes = append(nodes, node) if uint64(len(nodes)) == maxSharedNodes { break } } }() return encoding.WriteObject(conn, nodes) } // requestNodes is the calling end of the ShareNodes RPC. func (g *Gateway) requestNodes(conn modules.PeerConn) error { conn.SetDeadline(time.Now().Add(connStdDeadline)) var nodes []modules.NetAddress if err := encoding.ReadObject(conn, &nodes, maxSharedNodes*modules.MaxEncodedNetAddressLength); err != nil { return err } g.mu.Lock() for _, node := range nodes { err := g.addNode(node) if err != nil && err != errNodeExists && err != errOurAddress { g.log.Printf("WARN: peer '%v' sent the invalid addr '%v'", conn.RPCAddr(), node) } } err := g.save() if err != nil { g.log.Println("WARN: failed to save nodelist after requesting nodes:", err) } g.mu.Unlock() return nil } // permanentNodePurger is a thread that runs throughout the lifetime of the // gateway, purging unconnectable nodes from the node list in a sustainable // way. func (g *Gateway) permanentNodePurger(closeChan chan struct{}) { defer close(closeChan) for { // Start by sleeping for 10 minutes. This means that no nodes will be // purged for the first 10 minutes that Sia is running, and that any // failures to get nodes will be met by 10 minutes of waiting. // // At most one node will be contacted every 10 minutes. This minimizes // the total amount of keepalive traffic on the network. select { case <-time.After(nodePurgeDelay): case <-g.threads.StopChan(): // The gateway is shutting down, close out the thread. return } // Get a random node for scanning. g.mu.RLock() numNodes := len(g.nodes) node, err := g.randomNode() g.mu.RUnlock() if err == errNoNodes { // errNoNodes is a common error that will be resovled by the // bootstrap process. continue } else if err != nil { // Unusual error, create a logging statement. g.log.Println("ERROR: could not pick a random node for uptime check:", err) continue } if numNodes <= pruneNodeListLen { // There are not enough nodes in the gateway - pruning more is // probably a bad idea, and may affect the user's ability to // connect to the network in the future. continue } // Check whether this node is already a peer. If so, no need to dial // them. g.mu.RLock() _, exists := g.peers[node] g.mu.RUnlock() if exists { continue } // Try connecting to the random node. If the node is not reachable, // remove them from the node list. conn, err := g.dial(node) if err != nil { // NOTE: an error may be returned if the dial is cancelled // partway through, which would cause the node to be pruned // even though it may be a good node. Because nodes are // plentiful, that's not a huge problem. g.mu.Lock() g.removeNode(node) g.save() g.mu.Unlock() g.log.Debugf("INFO: removing node %q because dialing it failed: %v", node, err) continue } // If connection succeeds, supply an unacceptable version so that we // will not be added as a peer. // // NOTE: this is a somewhat clunky way of specifying that you didn't // actually want a connection. encoding.WriteObject(conn, "0.0.0") var reject string err = encoding.ReadObject(conn, &reject, build.MaxEncodedVersionLength) if err != nil { g.log.Debugln("ERROR: version handshake ping terminated unexpectedly:", err) } if reject != "reject" { g.log.Debugln("WARN: peer does not seem to have correctly rejected our ping:", reject) } conn.Close() } } // permanentNodeManager tries to keep the Gateway's node list healthy. As long // as the Gateway has fewer than healthyNodeListLen nodes, it asks a random // peer for more nodes. It also continually pings nodes in order to establish // their connectivity. Unresponsive nodes are aggressively removed. func (g *Gateway) permanentNodeManager(closeChan chan struct{}) { defer close(closeChan) for { // Wait 5 seconds so that a controlled number of node requests are made // to peers. select { case <-time.After(nodeListDelay): case <-g.threads.StopChan(): // Gateway is shutting down, close the thread. return } g.mu.RLock() numNodes := len(g.nodes) peer, err := g.randomOutboundPeer() g.mu.RUnlock() if err == errNoPeers { // errNoPeers is a common and expected error, there's no need to // log it. continue } else if err != nil { g.log.Println("ERROR: could not fetch a random peer:", err) continue } // Determine whether there are a satisfactory number of nodes in the // nodelist. If there are not, use the random peer from earlier to // expand the node list. if numNodes < healthyNodeListLen { err := g.managedRPC(peer, "ShareNodes", g.requestNodes) if err != nil { g.log.Debugf("WARN: RPC ShareNodes failed on peer %q: %v", peer, err) continue } } else { // There are enough nodes in the gateway, no need to check for more // every 5 seconds. Wait a while before checking again. select { case <-time.After(wellConnectedDelay): case <-g.threads.StopChan(): // Gateway is shutting down, close the thread. return } } } } Sia-1.0.4/modules/gateway/nodes_test.go000066400000000000000000000261471302033070500200420ustar00rootroot00000000000000package gateway import ( "strconv" "sync" "testing" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" ) const dummyNode = "111.111.111.111:1111" // TestAddNode tries adding a node to the gateway using the unexported addNode // function. Edge case trials are also performed. func TestAddNode(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestAddNode", t) defer g.Close() g.mu.Lock() defer g.mu.Unlock() if err := g.addNode(dummyNode); err != nil { t.Fatal("addNode failed:", err) } if err := g.addNode(dummyNode); err != errNodeExists { t.Error("addNode added duplicate node") } if err := g.addNode("foo"); err == nil { t.Error("addNode added unroutable address") } if err := g.addNode("foo:9981"); err == nil { t.Error("addNode added a non-IP address") } if err := g.addNode("[::]:9981"); err == nil { t.Error("addNode added unspecified address") } if err := g.addNode(g.myAddr); err != errOurAddress { t.Error("addNode added our own address") } } // TestRemoveNode tries remiving a node from the gateway. func TestRemoveNode(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestRemoveNode", t) defer g.Close() g.mu.Lock() defer g.mu.Unlock() if err := g.addNode(dummyNode); err != nil { t.Fatal("addNode failed:", err) } if err := g.removeNode(dummyNode); err != nil { t.Fatal("removeNode failed:", err) } if err := g.removeNode("bar"); err == nil { t.Fatal("removeNode removed nonexistent node") } } // TestRandomNode tries pulling random nodes from the gateway using // g.randomNode() under a variety of conditions. func TestRandomNode(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestRandomNode", t) defer g.Close() // Test with 0 nodes. g.mu.RLock() _, err := g.randomNode() g.mu.RUnlock() if err != errNoPeers { t.Fatal("randomNode should fail when the gateway has 0 nodes") } // Test with 1 node. g.mu.Lock() if err = g.addNode(dummyNode); err != nil { t.Fatal(err) } g.mu.Unlock() g.mu.RLock() addr, err := g.randomNode() g.mu.RUnlock() if err != nil { t.Fatal("randomNode failed:", err) } else if addr != dummyNode { t.Fatal("randomNode returned wrong address:", addr) } // Test again with 0 nodes. g.mu.Lock() err = g.removeNode(dummyNode) g.mu.Unlock() if err != nil { t.Fatal(err) } g.mu.RLock() _, err = g.randomNode() g.mu.RUnlock() if err != errNoPeers { t.Fatalf("randomNode returned wrong error: expected %v, got %v", errNoPeers, err) } // Test with 3 nodes. nodes := map[modules.NetAddress]int{ "111.111.111.111:1111": 0, "111.111.111.111:2222": 0, "111.111.111.111:3333": 0, } g.mu.Lock() for addr := range nodes { err := g.addNode(addr) if err != nil { t.Error(err) } } g.mu.Unlock() for i := 0; i < len(nodes)*10; i++ { g.mu.RLock() addr, err := g.randomNode() g.mu.RUnlock() if err != nil { t.Fatal("randomNode failed:", err) } nodes[addr]++ } for node, count := range nodes { if count == 0 { // 1-in-200000 chance of occurring naturally t.Errorf("node %v was never selected", node) } } } // TestShareNodes checks that two gateways will share nodes with eachother // following the desired sharing strategy. func TestShareNodes(t *testing.T) { if testing.Short() { t.SkipNow() } g1 := newTestingGateway("TestShareNodes1", t) defer g1.Close() g2 := newTestingGateway("TestShareNodes2", t) defer g2.Close() // add a node to g2 g2.mu.Lock() err := g2.addNode(dummyNode) g2.mu.Unlock() if err != nil { t.Fatal(err) } // connect err = g1.Connect(g2.Address()) if err != nil { t.Fatal("couldn't connect:", err) } // g1 should have received the node time.Sleep(100 * time.Millisecond) g1.mu.Lock() err = g1.addNode(dummyNode) g1.mu.Unlock() if err == nil { t.Fatal("gateway did not receive nodes during Connect:", g1.nodes) } // remove all nodes from both peers g1.mu.Lock() g1.nodes = map[modules.NetAddress]struct{}{} g1.mu.Unlock() g2.mu.Lock() g2.nodes = map[modules.NetAddress]struct{}{} g2.mu.Unlock() // SharePeers should now return no peers var nodes []modules.NetAddress err = g1.RPC(g2.Address(), "ShareNodes", func(conn modules.PeerConn) error { return encoding.ReadObject(conn, &nodes, maxSharedNodes*modules.MaxEncodedNetAddressLength) }) if err != nil { t.Fatal(err) } if len(nodes) != 0 { t.Fatal("gateway gave non-existent addresses:", nodes) } // sharing should be capped at maxSharedNodes for i := 1; i < int(maxSharedNodes)+11; i++ { g2.mu.Lock() err := g2.addNode(modules.NetAddress("111.111.111.111:" + strconv.Itoa(i))) g2.mu.Unlock() if err != nil { t.Fatal(err) } } err = g1.RPC(g2.Address(), "ShareNodes", func(conn modules.PeerConn) error { return encoding.ReadObject(conn, &nodes, maxSharedNodes*modules.MaxEncodedNetAddressLength) }) if err != nil { t.Fatal(err) } if uint64(len(nodes)) != maxSharedNodes { t.Fatalf("gateway gave wrong number of nodes: expected %v, got %v", maxSharedNodes, len(nodes)) } } // TestNodesAreSharedOnConnect tests that nodes that a gateway has never seen // before are added to the node list when connecting to another gateway that // has seen said nodes. func TestNodesAreSharedOnConnect(t *testing.T) { if testing.Short() { t.SkipNow() } g1 := newTestingGateway("TestNodesAreSharedOnConnect1", t) defer g1.Close() g2 := newTestingGateway("TestNodesAreSharedOnConnect2", t) defer g2.Close() g3 := newTestingGateway("TestNodesAreSharedOnConnect3", t) defer g3.Close() // connect g2 to g1 err := g2.Connect(g1.Address()) if err != nil { t.Fatal("couldn't connect:", err) } // connect g3 to g1 err = g3.Connect(g1.Address()) if err != nil { t.Fatal("couldn't connect:", err) } // g3 should have received g2's address from g1 time.Sleep(200 * time.Millisecond) g3.mu.Lock() defer g3.mu.Unlock() if _, ok := g3.nodes[g2.Address()]; !ok { t.Fatal("node was not relayed:", g3.nodes) } } // TestPruneNodeThreshold checks that gateways will not purge nodes if they are // below the purge threshold, even if those nodes are offline. func TestPruneNodeThreshold(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // The next part of the test expects the pruneNodeListLen to be at least // 'maxSharedNodes * 2 + 2 in size. if uint64(pruneNodeListLen) < (maxSharedNodes*2)+2 { t.Fatal("Constants do not match test, please either adjust the constants or refactor this test", maxSharedNodes, pruneNodeListLen) } // Create and connect pruneNodeListLen gateways. var gs []*Gateway for i := 0; i < pruneNodeListLen; i++ { gname := "TestPruneNodeThreshold" + strconv.Itoa(i) gs = append(gs, newTestingGateway(gname, t)) // Connect this gateway to the previous gateway. if i != 0 { err := gs[i].Connect(gs[i-1].myAddr) if err != nil { t.Fatal(err) } } } // Spin until all gateways have a nearly full node list. success := false for i := 0; i < 50; i++ { success = true for _, g := range gs { g.mu.RLock() gNodeLen := len(g.nodes) g.mu.RUnlock() if gNodeLen < pruneNodeListLen-2 { success = false break } } if !success { time.Sleep(time.Second * 1) } } if !success { t.Fatal("peers are not sharing nodes with eachother") } // Gateway node lists have been filled out. Take a bunch of gateways // offline and verify that they do not start pruning eachother. var wg sync.WaitGroup for i := 2; i < len(gs); i++ { wg.Add(1) go func(i int) { err := gs[i].Close() if err != nil { panic(err) } wg.Done() }(i) } wg.Wait() // Wait for 5 iterations of the node purge loop. Then verify that the // remaining gateways have not been purging nodes. time.Sleep(nodePurgeDelay * 5) // Check that the remaining gateways have not purged any nodes. gs[0].mu.RLock() gs0Nodes := len(gs[0].nodes) gs[0].mu.RUnlock() gs[1].mu.RLock() gs1Nodes := len(gs[1].nodes) gs[1].mu.RUnlock() if gs0Nodes < pruneNodeListLen-2 { t.Error("gateway seems to be pruning nodes below purge threshold") } if gs1Nodes < pruneNodeListLen-2 { t.Error("gateway seems to be pruning nodes below purge threshold") } // Close the remaining gateways. err := gs[0].Close() if err != nil { t.Error(err) } err = gs[1].Close() if err != nil { t.Error(err) } } // TestHealthyNodeListPruning checks that gateways will purge nodes if they are at // a healthy node threshold and the nodes are offline. func TestHealthyNodeListPruning(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // Create and connect healthyNodeListLen*2 gateways. var gs []*Gateway for i := 0; i < healthyNodeListLen*2; i++ { gname := "TestHealthyNodeListPruning" + strconv.Itoa(i) gs = append(gs, newTestingGateway(gname, t)) // Connect this gateway to the previous gateway. if i != 0 { err := gs[i].Connect(gs[i-1].myAddr) if err != nil { t.Fatal(err) } } // To help speed the test up, also connect this gateway to the peer two // back. if i > 1 { err := gs[i].Connect(gs[i-2].myAddr) if err != nil { t.Fatal(err) } } // To help speed the test up, also connect this gateway to a random // previous peer. if i > 2 { choice, err := crypto.RandIntn(i - 2) if err != nil { t.Fatal(err) } err = gs[i].Connect(gs[choice].myAddr) if err != nil { t.Fatal(err) } } } // Spin until all gateways have a nearly full node list. success := false for i := 0; i < 80; i++ { success = true for _, g := range gs { g.mu.RLock() gNodeLen := len(g.nodes) g.mu.RUnlock() if gNodeLen < healthyNodeListLen { success = false break } } if !success { time.Sleep(time.Second * 1) } } if !success { t.Fatal("peers are not sharing nodes with eachother") } // Gateway node lists have been filled out. Take a bunch of gateways // offline and verify that the remaining gateways begin pruning their // nodelist. var wg sync.WaitGroup for i := 2; i < len(gs); i++ { wg.Add(1) go func(i int) { err := gs[i].Close() if err != nil { panic(err) } wg.Done() }(i) } wg.Wait() // Wait for enough iterations of the node purge loop that over-pruning is // possible. (Over-pruning does not need to be guaranteed, causing this // test to fail once in a while is sufficient.) time.Sleep(nodePurgeDelay * time.Duration(healthyNodeListLen-pruneNodeListLen) * 12) // Check that the remaining gateways have pruned nodes. gs[0].mu.RLock() gs0Nodes := len(gs[0].nodes) gs[0].mu.RUnlock() gs[1].mu.RLock() gs1Nodes := len(gs[1].nodes) gs[1].mu.RUnlock() if gs0Nodes >= healthyNodeListLen-1 { t.Error("gateway is not pruning nodes", healthyNodeListLen, gs0Nodes) } if gs1Nodes >= healthyNodeListLen-1 { t.Error("gateway is not pruning nodes", healthyNodeListLen, gs1Nodes) } if gs0Nodes < pruneNodeListLen { t.Error("gateway is pruning too many nodes", gs0Nodes, pruneNodeListLen) } if gs1Nodes < pruneNodeListLen { t.Error("gateway is pruning too many nodes", gs1Nodes, pruneNodeListLen) } // Close the remaining gateways. err := gs[0].Close() if err != nil { t.Error(err) } err = gs[1].Close() if err != nil { t.Error(err) } } Sia-1.0.4/modules/gateway/peers.go000066400000000000000000000355711302033070500170120ustar00rootroot00000000000000package gateway import ( "errors" "fmt" "net" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/muxado" ) var ( errPeerExists = errors.New("already connected to this peer") errPeerRejectedConn = errors.New("peer rejected connection") ) // insufficientVersionError indicates a peer's version is insufficient. type insufficientVersionError string // Error implements the error interface for insufficientVersionError. func (s insufficientVersionError) Error() string { return "unacceptable version: " + string(s) } // invalidVersionError indicates a peer's version is not a valid version number. type invalidVersionError string // Error implements the error interface for invalidVersionError. func (s invalidVersionError) Error() string { return "invalid version: " + string(s) } type peer struct { modules.Peer sess muxado.Session } func (p *peer) open() (modules.PeerConn, error) { conn, err := p.sess.Open() if err != nil { return nil, err } return &peerConn{conn, p.NetAddress}, nil } func (p *peer) accept() (modules.PeerConn, error) { conn, err := p.sess.Accept() if err != nil { return nil, err } return &peerConn{conn, p.NetAddress}, nil } // addPeer adds a peer to the Gateway's peer list and spawns a listener thread // to handle its requests. func (g *Gateway) addPeer(p *peer) { g.peers[p.NetAddress] = p go g.threadedListenPeer(p) } // randomOutboundPeer returns a random outbound peer. func (g *Gateway) randomOutboundPeer() (modules.NetAddress, error) { // Get the list of outbound peers. var addrs []modules.NetAddress for addr, peer := range g.peers { if peer.Inbound { continue } addrs = append(addrs, addr) } if len(addrs) == 0 { return "", errNoPeers } // Of the remaining options, select one at random. r, err := crypto.RandIntn(len(addrs)) if err != nil { g.log.Severe("Random number generation failure:", err) } return addrs[r], nil } // permanentListen handles incoming connection requests. If the connection is // accepted, the peer will be added to the Gateway's peer list. func (g *Gateway) permanentListen(closeChan chan struct{}) { // Signal that the permanentListen thread has completed upon returning. defer close(closeChan) for { conn, err := g.listener.Accept() if err != nil { g.log.Debugln("[PL] Closing permanentListen:", err) return } go g.threadedAcceptConn(conn) // Sleep after each accept. This limits the rate at which the Gateway // will accept new connections. The intent here is to prevent new // incoming connections from kicking out old ones before they have a // chance to request additional nodes. select { case <-time.After(acceptInterval): case <-g.threads.StopChan(): return } } } // threadedAcceptConn adds a connecting node as a peer. func (g *Gateway) threadedAcceptConn(conn net.Conn) { if g.threads.Add() != nil { conn.Close() return } defer g.threads.Done() conn.SetDeadline(time.Now().Add(connStdDeadline)) addr := modules.NetAddress(conn.RemoteAddr().String()) g.log.Debugf("INFO: %v wants to connect", addr) remoteVersion, err := acceptConnVersionHandshake(conn, build.Version) if err != nil { g.log.Debugf("INFO: %v wanted to connect but version handshake failed: %v", addr, err) conn.Close() return } if build.VersionCmp(remoteVersion, handshakeUpgradeVersion) < 0 { err = g.managedAcceptConnOldPeer(conn, remoteVersion) } else { err = g.managedAcceptConnNewPeer(conn, remoteVersion) } if err != nil { g.log.Debugf("INFO: %v wanted to connect, but failed: %v", addr, err) conn.Close() return } // Handshake successful, remove the deadline. conn.SetDeadline(time.Time{}) g.log.Debugf("INFO: accepted connection from new peer %v (v%v)", addr, remoteVersion) } // managedAcceptConnOldPeer accepts a connection request from peers < v1.0.0. // The requesting peer is added as a peer, but is not added to the node list // (older peers do not share their dialback address). The peer is only added if // a nil error is returned. func (g *Gateway) managedAcceptConnOldPeer(conn net.Conn, remoteVersion string) error { addr := modules.NetAddress(conn.RemoteAddr().String()) g.mu.Lock() defer g.mu.Unlock() // Old peers are unable to give us a dialback port, so we can't verify // whether or not they are local peers. g.acceptPeer(&peer{ Peer: modules.Peer{ Inbound: true, Local: false, NetAddress: addr, Version: remoteVersion, }, sess: muxado.Server(conn), }) return nil } // managedAcceptConnNewPeer accepts connection requests from peers >= v1.0.0. // The requesting peer is added as a node and a peer. The peer is only added if // a nil error is returned. func (g *Gateway) managedAcceptConnNewPeer(conn net.Conn, remoteVersion string) error { // Learn the peer's dialback address. Peers older than v1.0.0 will only be // able to be discovered by newer peers via the ShareNodes RPC. remoteAddr, err := acceptConnPortHandshake(conn) if err != nil { return err } // Attempt to add the peer to the node list. If the add is successful and // the address is a local address, mark the peer as a local peer. local := false err = g.managedAddUntrustedNode(remoteAddr) if err != nil && remoteAddr.IsLocal() { local = true } g.mu.Lock() defer g.mu.Unlock() // Don't accept a connection from a peer we're already connected to. if _, exists := g.peers[remoteAddr]; exists { return fmt.Errorf("already connected to a peer on that address: %v", remoteAddr) } // Accept the peer. g.acceptPeer(&peer{ Peer: modules.Peer{ Inbound: true, Local: local, NetAddress: remoteAddr, Version: remoteVersion, }, sess: muxado.Server(conn), }) return nil } // acceptPeer makes room for the peer if necessary by kicking out existing // peers, then adds the peer to the peer list. func (g *Gateway) acceptPeer(p *peer) { // If we are not fully connected, add the peer without kicking any out. if len(g.peers) < fullyConnectedThreshold { g.addPeer(p) return } // Select a peer to kick. Outbound peers and local peers are not // available to be kicked. var addrs []modules.NetAddress for addr := range g.peers { // Do not kick outbound peers or local peers. if !p.Inbound || p.Local { continue } // Prefer kicking a peer with the same hostname. if addr.Host() == p.NetAddress.Host() { addrs = []modules.NetAddress{addr} break } addrs = append(addrs, addr) } if len(addrs) == 0 { // There is nobody suitable to kick, therefore do not kick anyone. g.addPeer(p) return } // Of the remaining options, select one at random. r, err := crypto.RandIntn(len(addrs)) if err != nil { g.log.Severe("random number generation failure:", err) } kick := addrs[r] g.peers[kick].sess.Close() delete(g.peers, kick) g.log.Printf("INFO: disconnected from %v to make room for %v\n", kick, p.NetAddress) g.addPeer(p) } // acceptConnPortHandshake performs the port handshake and should be called on // the side accepting a connection request. The remote address is only returned // if err == nil. func acceptConnPortHandshake(conn net.Conn) (remoteAddr modules.NetAddress, err error) { host, _, err := net.SplitHostPort(conn.RemoteAddr().String()) if err != nil { return "", err } // Read the peer's port that we can dial them back on. var dialbackPort string err = encoding.ReadObject(conn, &dialbackPort, 13) // Max port # is 65535 (5 digits long) + 8 byte string length prefix if err != nil { return "", fmt.Errorf("could not read remote peer's port: %v", err) } remoteAddr = modules.NetAddress(net.JoinHostPort(host, dialbackPort)) if err := remoteAddr.IsStdValid(); err != nil { return "", fmt.Errorf("peer's address (%v) is invalid: %v", remoteAddr, err) } // Sanity check to ensure that appending the port string to the host didn't // change the host. Only necessary because the peer sends the port as a string // instead of an integer. if remoteAddr.Host() != host { return "", fmt.Errorf("peer sent a port which modified the host") } return remoteAddr, nil } // connectPortHandshake performs the port handshake and should be called on the // side initiating the connection request. This shares our port with the peer // so they can connect to us in the future. func connectPortHandshake(conn net.Conn, port string) error { err := encoding.WriteObject(conn, port) if err != nil { return errors.New("could not write port #: " + err.Error()) } return nil } // acceptableVersion returns an error if the version is unacceptable. func acceptableVersion(version string) error { if !build.IsVersion(version) { return invalidVersionError(version) } if build.VersionCmp(version, minAcceptableVersion) < 0 { return insufficientVersionError(version) } return nil } // connectVersionHandshake performs the version handshake and should be called // on the side making the connection request. The remote version is only // returned if err == nil. func connectVersionHandshake(conn net.Conn, version string) (remoteVersion string, err error) { // Send our version. if err := encoding.WriteObject(conn, version); err != nil { return "", fmt.Errorf("failed to write version: %v", err) } // Read remote version. if err := encoding.ReadObject(conn, &remoteVersion, build.MaxEncodedVersionLength); err != nil { return "", fmt.Errorf("failed to read remote version: %v", err) } // Check that their version is acceptable. if remoteVersion == "reject" { return "", errPeerRejectedConn } if err := acceptableVersion(remoteVersion); err != nil { return "", err } return remoteVersion, nil } // acceptConnVersionHandshake performs the version handshake and should be // called on the side accepting a connection request. The remote version is // only returned if err == nil. func acceptConnVersionHandshake(conn net.Conn, version string) (remoteVersion string, err error) { // Read remote version. if err := encoding.ReadObject(conn, &remoteVersion, build.MaxEncodedVersionLength); err != nil { return "", fmt.Errorf("failed to read remote version: %v", err) } // Check that their version is acceptable. if err := acceptableVersion(remoteVersion); err != nil { if err := encoding.WriteObject(conn, "reject"); err != nil { return "", fmt.Errorf("failed to write reject: %v", err) } return "", err } // Send our version. if err := encoding.WriteObject(conn, version); err != nil { return "", fmt.Errorf("failed to write version: %v", err) } return remoteVersion, nil } // managedConnectOldPeer connects to peers < v1.0.0. The peer is added as a // node and a peer. The peer is only added if a nil error is returned. func (g *Gateway) managedConnectOldPeer(conn net.Conn, remoteVersion string, remoteAddr modules.NetAddress) error { // Attempt to add the peer to the node list. If the add is successful and // the address is a local address, mark the peer as a local peer. local := false err := g.managedAddUntrustedNode(remoteAddr) if err != nil && remoteAddr.IsLocal() { local = true } g.mu.Lock() defer g.mu.Unlock() g.addPeer(&peer{ Peer: modules.Peer{ Inbound: false, Local: local, NetAddress: remoteAddr, Version: remoteVersion, }, sess: muxado.Client(conn), }) return nil } // managedConnectNewPeer connects to peers >= v1.0.0. The peer is added as a // node and a peer. The peer is only added if a nil error is returned. func (g *Gateway) managedConnectNewPeer(conn net.Conn, remoteVersion string, remoteAddr modules.NetAddress) error { g.mu.RLock() port := g.port g.mu.RUnlock() // Send our dialable address to the peer so they can dial us back should we // disconnect. err := connectPortHandshake(conn, port) if err != nil { return err } // Attempt to add the peer to the node list. If the add is successful and // the address is a local address, mark the peer as a local peer. local := false err = g.managedAddUntrustedNode(remoteAddr) if err != nil && remoteAddr.IsLocal() { local = true } g.mu.Lock() defer g.mu.Unlock() g.addPeer(&peer{ Peer: modules.Peer{ Inbound: false, Local: local, NetAddress: remoteAddr, Version: remoteVersion, }, sess: muxado.Client(conn), }) return nil } // managedConnect establishes a persistent connection to a peer, and adds it to // the Gateway's peer list. func (g *Gateway) managedConnect(addr modules.NetAddress) error { // Perform verification on the input address. g.mu.RLock() gaddr := g.myAddr g.mu.RUnlock() if addr == gaddr { return errors.New("can't connect to our own address") } if err := addr.IsStdValid(); err != nil { return errors.New("can't connect to invalid address") } if net.ParseIP(addr.Host()) == nil { return errors.New("address must be an IP address") } g.mu.RLock() _, exists := g.peers[addr] g.mu.RUnlock() if exists { return errPeerExists } // Dial the peer and perform peer initialization. conn, err := g.dial(addr) if err != nil { return err } // Perform peer initialization. remoteVersion, err := connectVersionHandshake(conn, build.Version) if err != nil { conn.Close() return err } if build.VersionCmp(remoteVersion, handshakeUpgradeVersion) < 0 { err = g.managedConnectOldPeer(conn, remoteVersion, addr) } else { err = g.managedConnectNewPeer(conn, remoteVersion, addr) } if err != nil { conn.Close() return err } g.log.Debugln("INFO: connected to new peer", addr) // Connection successful, clear the timeout as to maintain a persistent // connection to this peer. conn.SetDeadline(time.Time{}) // call initRPCs g.mu.RLock() for name, fn := range g.initRPCs { go func(name string, fn modules.RPCFunc) { if g.threads.Add() != nil { return } defer g.threads.Done() err := g.managedRPC(addr, name, fn) if err != nil { g.log.Debugf("INFO: RPC %q on peer %q failed: %v", name, addr, err) } }(name, fn) } g.mu.RUnlock() return nil } // Connect establishes a persistent connection to a peer, and adds it to the // Gateway's peer list. func (g *Gateway) Connect(addr modules.NetAddress) error { if err := g.threads.Add(); err != nil { return err } defer g.threads.Done() return g.managedConnect(addr) } // Disconnect terminates a connection to a peer and removes it from the // Gateway's peer list. The peer's address remains in the node list. func (g *Gateway) Disconnect(addr modules.NetAddress) error { if err := g.threads.Add(); err != nil { return err } defer g.threads.Done() g.mu.RLock() p, exists := g.peers[addr] g.mu.RUnlock() if !exists { return errors.New("not connected to that node") } g.mu.Lock() delete(g.peers, addr) g.mu.Unlock() if err := p.sess.Close(); err != nil { return err } g.log.Println("INFO: disconnected from peer", addr) return nil } // Peers returns the addresses currently connected to the Gateway. func (g *Gateway) Peers() []modules.Peer { g.mu.RLock() defer g.mu.RUnlock() var peers []modules.Peer for _, p := range g.peers { peers = append(peers, p.Peer) } return peers } Sia-1.0.4/modules/gateway/peers_test.go000066400000000000000000000533451302033070500200500ustar00rootroot00000000000000package gateway import ( "net" "strconv" "testing" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/muxado" ) // dummyConn implements the net.Conn interface, but does not carry any actual // data. It is passed to muxado, because passing nil results in segfaults. type dummyConn struct { net.Conn } // muxado uses these methods when sending its GoAway signal func (dc *dummyConn) Write(p []byte) (int, error) { return len(p), nil } // Close is a no-op to satisfy the Conn interface. func (dc *dummyConn) Close() error { return nil } // SetWriteDeadline is a no-op to satisfy the Conn interface. func (dc *dummyConn) SetWriteDeadline(time.Time) error { return nil } // TestAddPeer tries adding a peer to the gateway. func TestAddPeer(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestAddPeer", t) defer g.Close() g.mu.Lock() defer g.mu.Unlock() g.addPeer(&peer{ Peer: modules.Peer{ NetAddress: "foo.com:123", }, sess: muxado.Client(new(dummyConn)), }) if len(g.peers) != 1 { t.Fatal("gateway did not add peer") } } // TestRandomInbountPeer checks that randomOutboundPeer returns the correct // peer. func TestRandomOutboundPeer(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestRandomInboundPeer", t) defer g.Close() g.mu.Lock() defer g.mu.Unlock() _, err := g.randomOutboundPeer() if err != errNoPeers { t.Fatal("expected errNoPeers, got", err) } g.addPeer(&peer{ Peer: modules.Peer{ NetAddress: "foo.com:123", Inbound: false, }, sess: muxado.Client(new(dummyConn)), }) if len(g.peers) != 1 { t.Fatal("gateway did not add peer") } addr, err := g.randomOutboundPeer() if err != nil || addr != "foo.com:123" { t.Fatal("gateway did not select random peer") } } // TestListen is a general test probling the connection listener. func TestListen(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestListen", t) defer g.Close() // compliant connect with old version conn, err := net.Dial("tcp", string(g.Address())) if err != nil { t.Fatal("dial failed:", err) } addr := modules.NetAddress(conn.LocalAddr().String()) ack, err := connectVersionHandshake(conn, "0.1") if err != errPeerRejectedConn { t.Fatal(err) } if ack != "" { t.Fatal("gateway should have rejected old version") } for i := 0; i < 10; i++ { g.mu.RLock() _, ok := g.peers[addr] g.mu.RUnlock() if ok { t.Fatal("gateway should not have added an old peer") } time.Sleep(20 * time.Millisecond) } // a simple 'conn.Close' would not obey the muxado disconnect protocol muxado.Client(conn).Close() // compliant connect with invalid port conn, err = net.Dial("tcp", string(g.Address())) if err != nil { t.Fatal("dial failed:", err) } addr = modules.NetAddress(conn.LocalAddr().String()) ack, err = connectVersionHandshake(conn, build.Version) if err != nil { t.Fatal(err) } if ack != build.Version { t.Fatal("gateway should have given ack") } err = connectPortHandshake(conn, "0") if err != nil { t.Fatal(err) } for i := 0; i < 10; i++ { g.mu.RLock() _, ok := g.peers[addr] g.mu.RUnlock() if ok { t.Fatal("gateway should not have added a peer with an invalid port") } time.Sleep(20 * time.Millisecond) } // a simple 'conn.Close' would not obey the muxado disconnect protocol muxado.Client(conn).Close() // compliant connect conn, err = net.Dial("tcp", string(g.Address())) if err != nil { t.Fatal("dial failed:", err) } addr = modules.NetAddress(conn.LocalAddr().String()) ack, err = connectVersionHandshake(conn, build.Version) if err != nil { t.Fatal(err) } if ack != build.Version { t.Fatal("gateway should have given ack") } err = connectPortHandshake(conn, addr.Port()) if err != nil { t.Fatal(err) } // g should add the peer var ok bool for !ok { g.mu.RLock() _, ok = g.peers[addr] g.mu.RUnlock() } muxado.Client(conn).Close() // g should remove the peer for ok { g.mu.RLock() _, ok = g.peers[addr] g.mu.RUnlock() } // uncompliant connect conn, err = net.Dial("tcp", string(g.Address())) if err != nil { t.Fatal("dial failed:", err) } if _, err := conn.Write([]byte("missing length prefix")); err != nil { t.Fatal("couldn't write malformed header") } // g should have closed the connection if n, err := conn.Write([]byte("closed")); err != nil && n > 0 { t.Error("write succeeded after closed connection") } } // TestConnect verifies that connecting peers will add peer relationships to // the gateway, and that certain edge cases are properly handled. func TestConnect(t *testing.T) { if testing.Short() { t.SkipNow() } // create bootstrap peer bootstrap := newTestingGateway("TestConnect1", t) defer bootstrap.Close() // give it a node bootstrap.mu.Lock() bootstrap.addNode(dummyNode) bootstrap.mu.Unlock() // create peer who will connect to bootstrap g := newTestingGateway("TestConnect2", t) defer g.Close() // first simulate a "bad" connect, where bootstrap won't share its nodes bootstrap.mu.Lock() bootstrap.handlers[handlerName("ShareNodes")] = func(modules.PeerConn) error { return nil } bootstrap.mu.Unlock() // connect err := g.Connect(bootstrap.Address()) if err != nil { t.Fatal(err) } // g should not have the node if g.removeNode(dummyNode) == nil { t.Fatal("bootstrapper should not have received dummyNode:", g.nodes) } // split 'em up g.Disconnect(bootstrap.Address()) bootstrap.Disconnect(g.Address()) // now restore the correct ShareNodes RPC and try again bootstrap.mu.Lock() bootstrap.handlers[handlerName("ShareNodes")] = bootstrap.shareNodes bootstrap.mu.Unlock() err = g.Connect(bootstrap.Address()) if err != nil { t.Fatal(err) } // g should have the node time.Sleep(200 * time.Millisecond) g.mu.RLock() if _, ok := g.nodes[dummyNode]; !ok { g.mu.RUnlock() // Needed to prevent a deadlock if this error condition is reached. t.Fatal("bootstrapper should have received dummyNode:", g.nodes) } g.mu.RUnlock() } // TestUnitAcceptableVersion tests that the acceptableVersion func returns an // error for unacceptable versions. func TestUnitAcceptableVersion(t *testing.T) { invalidVersions := []string{ // ascii gibberish "foobar", "foobar.0", "foobar.9", "0.foobar", "9.foobar", "foobar.0.0", "foobar.9.9", "0.foobar.0", "9.foobar.9", "0.0.foobar", "9.9.foobar", // utf-8 gibberish "世界", "世界.0", "世界.9", "0.世界", "9.世界", "世界.0.0", "世界.9.9", "0.世界.0", "9.世界.9", "0.0.世界", "9.9.世界", // missing numbers ".", "..", "...", "0.", ".1", "2..", ".3.", "..4", "5.6.", ".7.8", ".9.0.", } for _, v := range invalidVersions { err := acceptableVersion(v) if _, ok := err.(invalidVersionError); err == nil || !ok { t.Errorf("acceptableVersion returned %q for version %q, but expected invalidVersionError", err, v) } } insufficientVersions := []string{ // random small versions "0", "00", "0000000000", "0.0", "0000000000.0", "0.0000000000", "0.0.0.0.0.0.0.0", "0.0.9", "0.0.999", "0.0.99999999999", "0.1.2", "0.1.2.3.4.5.6.7.8.9", // pre-hardfork versions "0.3.3", "0.3.9.9.9.9.9.9.9.9.9.9", "0.3.9999999999", } for _, v := range insufficientVersions { err := acceptableVersion(v) if _, ok := err.(insufficientVersionError); err == nil || !ok { t.Errorf("acceptableVersion returned %q for version %q, but expected insufficientVersionError", err, v) } } validVersions := []string{ minAcceptableVersion, "0.4.0", "0.6.0", "0.6.1", "0.9", "0.999", "0.9999999999", "1", "1.0", "1.0.0", "9", "9.0", "9.0.0", "9.9.9", } for _, v := range validVersions { err := acceptableVersion(v) if err != nil { t.Errorf("acceptableVersion returned %q for version %q, but expected nil", err, v) } } } // TestConnectRejectsInvalidAddrs tests that Connect only connects to valid IP // addresses. func TestConnectRejectsInvalidAddrs(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestConnectRejectsInvalidAddrs", t) defer g.Close() g2 := newTestingGateway("TestConnectRejectsInvalidAddrs2", t) defer g2.Close() _, g2Port, err := net.SplitHostPort(string(g2.Address())) if err != nil { t.Fatal(err) } tests := []struct { addr modules.NetAddress wantErr bool msg string }{ { addr: "127.0.0.1:123", wantErr: true, msg: "Connect should reject unreachable addresses", }, { addr: "111.111.111.111:0", wantErr: true, msg: "Connect should reject invalid NetAddresses", }, { addr: modules.NetAddress(net.JoinHostPort("localhost", g2Port)), wantErr: true, msg: "Connect should reject non-IP addresses", }, { addr: g2.Address(), msg: "Connect failed to connect to another gateway", }, { addr: g2.Address(), wantErr: true, msg: "Connect should reject an address it's already connected to", }, } for _, tt := range tests { err := g.Connect(tt.addr) if tt.wantErr != (err != nil) { t.Errorf("%v, wantErr: %v, err: %v", tt.msg, tt.wantErr, err) } } } // TestConnectRejectsVersions tests that Gateway.Connect only accepts peers // with sufficient and valid versions. func TestConnectRejectsVersions(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestConnectRejectsVersions", t) defer g.Close() // Setup a listener that mocks Gateway.acceptConn, but sends the // version sent over mockVersionChan instead of build.Version. listener, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatal(err) } defer listener.Close() tests := []struct { version string errWant error invalidVersion bool insufficientVersion bool msg string }{ // Test that Connect fails when the remote peer's version is "reject". { version: "reject", errWant: errPeerRejectedConn, msg: "Connect should fail when the remote peer rejects the connection", }, // Test that Connect fails when the remote peer's version is ascii gibberish. { version: "foobar", invalidVersion: true, msg: "Connect should fail when the remote peer's version is ascii gibberish", }, // Test that Connect fails when the remote peer's version is utf8 gibberish. { version: "世界", invalidVersion: true, msg: "Connect should fail when the remote peer's version is utf8 gibberish", }, // Test that Connect fails when the remote peer's version is < 0.4.0 (0). { version: "0", insufficientVersion: true, msg: "Connect should fail when the remote peer's version is 0", }, { version: "0.0.0", insufficientVersion: true, msg: "Connect should fail when the remote peer's version is 0.0.0", }, { version: "0000.0000.0000", insufficientVersion: true, msg: "Connect should fail when the remote peer's version is 0000.0000.0000", }, { version: "0.3.9", insufficientVersion: true, msg: "Connect should fail when the remote peer's version is 0.3.9", }, { version: "0.3.9999", insufficientVersion: true, msg: "Connect should fail when the remote peer's version is 0.3.9999", }, { version: "0.3.9.9.9", insufficientVersion: true, msg: "Connect should fail when the remote peer's version is 0.3.9.9.9", }, // Test that Connect succeeds when the remote peer's version is 0.4.0. { version: "0.4.0", msg: "Connect should succeed when the remote peer's version is 0.4.0", }, // Test that Connect succeeds when the remote peer's version is > 0.4.0. { version: "9", msg: "Connect should succeed when the remote peer's version is 9", }, { version: "9.9.9", msg: "Connect should succeed when the remote peer's version is 9.9.9", }, { version: "9999.9999.9999", msg: "Connect should succeed when the remote peer's version is 9999.9999.9999", }, } for _, tt := range tests { doneChan := make(chan struct{}) go func() { defer close(doneChan) conn, err := listener.Accept() if err != nil { panic(err) } remoteVersion, err := acceptConnVersionHandshake(conn, tt.version) if err != nil { panic(err) } if remoteVersion != build.Version { panic("remoteVersion != build.Version") } }() err = g.Connect(modules.NetAddress(listener.Addr().String())) switch { case tt.invalidVersion: // Check that the error is the expected type. if _, ok := err.(invalidVersionError); !ok { t.Fatalf("expected Connect to error with invalidVersionError: %s", tt.msg) } case tt.insufficientVersion: // Check that the error is the expected type. if _, ok := err.(insufficientVersionError); !ok { t.Fatalf("expected Connect to error with insufficientVersionError: %s", tt.msg) } default: // Check that the error is the expected error. if err != tt.errWant { t.Fatalf("expected Connect to error with '%v', but got '%v': %s", tt.errWant, err, tt.msg) } } <-doneChan g.Disconnect(modules.NetAddress(listener.Addr().String())) } } // TestAcceptConnRejectsVersions tests that Gateway.acceptConn only accepts // peers with sufficient and valid versions. func TestAcceptConnRejectsVersions(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestAcceptConnRejectsVersions", t) defer g.Close() tests := []struct { remoteVersion string versionResponseWant string errWant error msg string }{ // Test that acceptConn fails when the remote peer's version is "reject". { remoteVersion: "reject", versionResponseWant: "", errWant: errPeerRejectedConn, msg: "acceptConn shouldn't accept a remote peer whose version is \"reject\"", }, // Test that acceptConn fails when the remote peer's version is ascii gibberish. { remoteVersion: "foobar", versionResponseWant: "", errWant: errPeerRejectedConn, msg: "acceptConn shouldn't accept a remote peer whose version is ascii gibberish", }, // Test that acceptConn fails when the remote peer's version is utf8 gibberish. { remoteVersion: "世界", versionResponseWant: "", errWant: errPeerRejectedConn, msg: "acceptConn shouldn't accept a remote peer whose version is utf8 gibberish", }, // Test that acceptConn fails when the remote peer's version is < 0.4.0 (0). { remoteVersion: "0", versionResponseWant: "", errWant: errPeerRejectedConn, msg: "acceptConn shouldn't accept a remote peer whose version is 0", }, { remoteVersion: "0.0.0", versionResponseWant: "", errWant: errPeerRejectedConn, msg: "acceptConn shouldn't accept a remote peer whose version is 0.0.0", }, { remoteVersion: "0000.0000.0000", versionResponseWant: "", errWant: errPeerRejectedConn, msg: "acceptConn shouldn't accept a remote peer whose version is 0000.000.000", }, { remoteVersion: "0.3.9", versionResponseWant: "", errWant: errPeerRejectedConn, msg: "acceptConn shouldn't accept a remote peer whose version is 0.3.9", }, { remoteVersion: "0.3.9999", versionResponseWant: "", errWant: errPeerRejectedConn, msg: "acceptConn shouldn't accept a remote peer whose version is 0.3.9999", }, { remoteVersion: "0.3.9.9.9", versionResponseWant: "", errWant: errPeerRejectedConn, msg: "acceptConn shouldn't accept a remote peer whose version is 0.3.9.9.9", }, // Test that acceptConn succeeds when the remote peer's version is 0.4.0. { remoteVersion: "0.4.0", versionResponseWant: build.Version, msg: "acceptConn should accept a remote peer whose version is 0.4.0", }, // Test that acceptConn succeeds when the remote peer's version is > 0.4.0. { remoteVersion: "9", versionResponseWant: build.Version, msg: "acceptConn should accept a remote peer whose version is 9", }, { remoteVersion: "9.9.9", versionResponseWant: build.Version, msg: "acceptConn should accept a remote peer whose version is 9.9.9", }, { remoteVersion: "9999.9999.9999", versionResponseWant: build.Version, msg: "acceptConn should accept a remote peer whose version is 9999.9999.9999", }, } for _, tt := range tests { conn, err := net.DialTimeout("tcp", string(g.Address()), dialTimeout) if err != nil { t.Fatal(err) } remoteVersion, err := connectVersionHandshake(conn, tt.remoteVersion) if err != tt.errWant { t.Fatal(err) } if remoteVersion != tt.versionResponseWant { t.Fatal(tt.msg) } conn.Close() } } // TestDisconnect checks that calls to gateway.Disconnect correctly disconnect // and remove peers from the gateway. func TestDisconnect(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestDisconnect", t) defer g.Close() if err := g.Disconnect("bar.com:123"); err == nil { t.Fatal("disconnect removed unconnected peer") } // dummy listener to accept connection l, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatal("couldn't start listener:", err) } go func() { _, err := l.Accept() if err != nil { panic(err) } }() // skip standard connection protocol conn, err := net.Dial("tcp", l.Addr().String()) if err != nil { t.Fatal("dial failed:", err) } g.mu.Lock() g.addPeer(&peer{ Peer: modules.Peer{ NetAddress: "foo.com:123", }, sess: muxado.Client(conn), }) g.mu.Unlock() if err := g.Disconnect("foo.com:123"); err != nil { t.Fatal("disconnect failed:", err) } } // TestPeerManager checks that the peer manager is properly spacing out peer // connection requests. func TestPeerManager(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() g1 := newTestingGateway("TestPeerManager1", t) defer g1.Close() // create a valid node to connect to g2 := newTestingGateway("TestPeerManager2", t) defer g2.Close() // g1's node list should only contain g2 g1.mu.Lock() g1.nodes = map[modules.NetAddress]struct{}{} g1.nodes[g2.Address()] = struct{}{} g1.mu.Unlock() // when peerManager wakes up, it should connect to g2. time.Sleep(time.Second + noNodesDelay) g1.mu.RLock() defer g1.mu.RUnlock() if len(g1.peers) != 1 || g1.peers[g2.Address()] == nil { t.Fatal("gateway did not connect to g2:", g1.peers) } } // TestOverloadedBootstrap creates a bunch of gateways and connects all of them // to the first gateway, the bootstrap gateway. More gateways will be created // than is allowed by the bootstrap for the total number of connections. After // waiting, all peers should eventually get to the full number of outbound // peers. func TestOverloadedBootstrap(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // Create fullyConnectedThreshold*2 peers and connect them all to only the // first node. var gs []*Gateway for i := 0; i < fullyConnectedThreshold*2; i++ { gname := "TestOverloadedBootstrap" + strconv.Itoa(i) gs = append(gs, newTestingGateway(gname, t)) // Connect this gateway to the first gateway. if i == 0 { continue } err := gs[i].Connect(gs[0].myAddr) for j := 0; j < 100 && err != nil; j++ { time.Sleep(time.Millisecond * 250) err = gs[i].Connect(gs[0].myAddr) } if err != nil { panic(err) } } // Spin until all gateways have a complete number of outbound peers. success := false for i := 0; i < 100; i++ { success = true for _, g := range gs { outboundPeers := 0 g.mu.RLock() for _, p := range g.peers { if !p.Inbound { outboundPeers++ } } g.mu.RUnlock() if outboundPeers < wellConnectedThreshold { success = false break } } if !success { time.Sleep(time.Second) } } if !success { for i, g := range gs { outboundPeers := 0 g.mu.RLock() for _, p := range g.peers { if !p.Inbound { outboundPeers++ } } g.mu.RUnlock() t.Log("Gateway", i, ":", outboundPeers) } t.Fatal("after 100 seconds not all gateways able to become well connected") } // Randomly close many of the peers. For many peers, this should put them // below the well connected threshold, but there are still enough nodes on // the network that no partitions should occur. var newGS []*Gateway perm, err := crypto.Perm(len(gs)) if err != nil { t.Fatal(err) } // Reorder the gateways. for _, i := range perm { newGS = append(newGS, gs[i]) } cutSize := len(newGS) / 4 // Close the first many of the now-randomly-sorted gateways. for _, g := range newGS[:cutSize] { err := g.Close() if err != nil { t.Fatal(err) } } // Set 'gs' equal to the remaining gateways. gs = newGS[cutSize:] // Spin until all gateways have a complete number of outbound peers. The // test can fail if there are network partitions, however not a huge // magnitude of nodes are being removed, and they all started with 4 // connections. A partition is unlikely. success = false for i := 0; i < 100; i++ { success = true for _, g := range gs { outboundPeers := 0 g.mu.RLock() for _, p := range g.peers { if !p.Inbound { outboundPeers++ } } g.mu.RUnlock() if outboundPeers < wellConnectedThreshold { success = false break } } if !success { time.Sleep(time.Second) } } if !success { t.Fatal("after 100 seconds not all gateways able to become well connected") } // Close all remaining gateways. for _, g := range gs { err = g.Close() if err != nil { t.Error(err) } } } Sia-1.0.4/modules/gateway/peersmanager.go000066400000000000000000000113651302033070500203400ustar00rootroot00000000000000package gateway import ( "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" ) // managedPeerManagerConnect is a blocking function which tries to connect to // the input addreess as a peer. func (g *Gateway) managedPeerManagerConnect(addr modules.NetAddress) { g.log.Debugln("[PMC] Attempting connection to", addr) err := g.managedConnect(addr) if err == errPeerExists { // This peer is already connected to us. Safety around the // oubound peers relates to the fact that we have picked out // the outbound peers instead of allow the attacker to pick out // the peers for us. Because we have made the selection, it is // okay to set the peer as an outbound peer. // // The nodelist size check ensures that an attacker can't flood // a new node with a bunch of inbound requests. Doing so would // result in a nodelist that's entirely full of attacker nodes. // There's not much we can do about that anyway, but at least // we can hold off making attacker nodes 'outbound' peers until // our nodelist has had time to fill up naturally. g.mu.Lock() p, exists := g.peers[addr] if exists { // Have to check it exists because we released the lock, a // race condition could mean that the peer was disconnected // before this code block was reached. p.Inbound = false g.log.Debugln("[PMC] [SUCCESS] existing peer has been converted to outbound peer:", addr) } else { g.log.Debugln("[PMC] errPeerExists was returned, but by the time the peer was to be set to 'outbound', it no longer existed.") } g.mu.Unlock() } else if err != nil { g.log.Debugf("[PMC] WARN: automatic connect to %v failed: %v\n", addr, err) } else { g.log.Debugln("[PMC] [SUCCESS] addr has been successfully added to the gateway:", addr) } } // numOutboundPeers returns the number of outbound peers in the gateway. func (g *Gateway) numOutboundPeers() (numOutboundPeers int) { g.mu.RLock() defer g.mu.RUnlock() for _, p := range g.peers { if !p.Inbound { numOutboundPeers++ } } return numOutboundPeers } // permanentPeerManager tries to keep the Gateway well-connected. As long as // the Gateway is not well-connected, it tries to connect to random nodes. func (g *Gateway) permanentPeerManager(closedChan chan struct{}) { // Send a signal upon shutdown. defer close(closedChan) // permanentPeerManager will attempt to connect to peers asynchronously, // such that multiple connection attempts can be open at once, but a // limited number. connectionLimiterChan := make(chan struct{}, maxConcurrentOutboundPeerRequests) for { // If the gateway is well connected, sleep for a while and then try // again. numOutboundPeers := g.numOutboundPeers() if numOutboundPeers >= wellConnectedThreshold { if !g.managedSleep(wellConnectedDelay) { return } continue } // Fetch a random node. g.mu.RLock() addr, err := g.randomNode() g.mu.RUnlock() // If there was an error, log the error and then wait a while before // trying again. if err != nil { g.log.Debugln("[PPM] Unable to acquire selected peer:", err) if !g.managedSleep(noNodesDelay) { return } continue } // We need at least some of our outbound peers to be remote peers. If // we already have reached a certain threshold of outbound peers and // this peer is a local peer, do not consider it for an outbound peer. // Sleep breifly to prevent the gateway from hogging the CPU if all // peers are local. if numOutboundPeers >= maxLocalOutboundPeers && addr.IsLocal() && build.Release != "testing" { g.log.Debugln("[PPM] Ignorning selected peer; this peer is local and we already have multiple outbound peers:", addr) if !g.managedSleep(unwantedLocalPeerDelay) { return } continue } g.log.Debugln("[PPM] Gateway does not have enough peers, attempting to acquire a new one:", addr) // Try connecting to that peer in a goroutine. Do not block unless // there are currently 3 or more peer connection attempts open at once. // Before spawning the thread, make sure that there is enough room by // throwing a struct into the buffered channel. connectionLimiterChan <- struct{}{} go func(addr modules.NetAddress) { // After completion, take the struct out of the channel so that the // next thread may proceed. defer func() { <-connectionLimiterChan }() if err := g.threads.Add(); err != nil { return } defer g.threads.Done() // peerManagerConnect will handle all of its own logging. g.managedPeerManagerConnect(addr) }(addr) // Wait a bit before trying the next peer. The peer connections are // non-blocking, so they should be spaced out to avoid spinning up an // uncontrolled number of threads and therefore peer connections. if !g.managedSleep(acquiringPeersDelay) { return } } } Sia-1.0.4/modules/gateway/persist.go000066400000000000000000000030611302033070500173520ustar00rootroot00000000000000package gateway import ( "path/filepath" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" ) const ( // nodesFile is the name of the file that contains all seen nodes. nodesFile = "nodes.json" // logFile is the name of the log file. logFile = modules.GatewayDir + ".log" ) // persistMetadata contains the header and version strings that identify the // gateway persist file. var persistMetadata = persist.Metadata{ Header: "Sia Node List", Version: "0.3.3", } // persistData returns the data in the Gateway that will be saved to disk. func (g *Gateway) persistData() (nodes []modules.NetAddress) { for node := range g.nodes { nodes = append(nodes, node) } return } // load loads the Gateway's persistent data from disk. func (g *Gateway) load() error { var nodes []modules.NetAddress err := persist.LoadFile(persistMetadata, &nodes, filepath.Join(g.persistDir, nodesFile)) if err != nil { return err } for _, node := range nodes { err := g.addNode(node) if err != nil { g.log.Printf("WARN: error loading node '%v' from persist: %v", node, err) } } return nil } // save stores the Gateway's persistent data on disk. func (g *Gateway) save() error { return persist.SaveFile(persistMetadata, g.persistData(), filepath.Join(g.persistDir, nodesFile)) } // saveSync stores the Gateway's persistent data on disk, and then syncs to // disk to minimize the possibility of data loss. func (g *Gateway) saveSync() error { return persist.SaveFileSync(persistMetadata, g.persistData(), filepath.Join(g.persistDir, nodesFile)) } Sia-1.0.4/modules/gateway/persist_test.go000066400000000000000000000006341302033070500204140ustar00rootroot00000000000000package gateway import ( "testing" ) func TestLoad(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestLoad", t) g.mu.Lock() g.addNode(dummyNode) g.save() g.mu.Unlock() g.Close() g2, err := New("localhost:0", false, g.persistDir) if err != nil { t.Fatal(err) } if _, ok := g2.nodes[dummyNode]; !ok { t.Fatal("gateway did not load old peer list:", g2.nodes) } } Sia-1.0.4/modules/gateway/rpc.go000066400000000000000000000166241302033070500164560ustar00rootroot00000000000000package gateway import ( "errors" "sync" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" ) // rpcID is an 8-byte signature that is added to all RPCs to tell the gatway // what to do with the RPC. type rpcID [8]byte // String returns a string representation of an rpcID. Empty elements of rpcID // will be encoded as spaces. func (id rpcID) String() string { for i := range id { if id[i] == 0 { id[i] = ' ' } } return string(id[:]) } // handlerName truncates a string to 8 bytes. If len(name) < 8, the remaining // bytes are 0. A handlerName is specified at the beginning of each network // call, indicating which function should handle the connection. func handlerName(name string) (id rpcID) { copy(id[:], name) return } // managedRPC calls an RPC on the given address. managedRPC cannot be called on // an address that the Gateway is not connected to. func (g *Gateway) managedRPC(addr modules.NetAddress, name string, fn modules.RPCFunc) error { g.mu.RLock() peer, ok := g.peers[addr] g.mu.RUnlock() if !ok { return errors.New("can't call RPC on unconnected peer " + string(addr)) } conn, err := peer.open() if err != nil { return err } defer conn.Close() // write header if err := encoding.WriteObject(conn, handlerName(name)); err != nil { return err } // call fn return fn(conn) } // RPC calls an RPC on the given address. RPC cannot be called on an address // that the Gateway is not connected to. func (g *Gateway) RPC(addr modules.NetAddress, name string, fn modules.RPCFunc) error { if err := g.threads.Add(); err != nil { return err } defer g.threads.Done() return g.managedRPC(addr, name, fn) } // RegisterRPC registers an RPCFunc as a handler for a given identifier. To // call an RPC, use gateway.RPC, supplying the same identifier given to // RegisterRPC. Identifiers should always use PascalCase. The first 8 // characters of an identifier should be unique, as the identifier used // internally is truncated to 8 bytes. func (g *Gateway) RegisterRPC(name string, fn modules.RPCFunc) { g.mu.Lock() defer g.mu.Unlock() if _, ok := g.handlers[handlerName(name)]; ok { build.Critical("RPC already registered: " + name) } g.handlers[handlerName(name)] = fn } // UnregisterRPC unregisters an RPC and removes the corresponding RPCFunc from // g.handlers. Future calls to the RPC by peers will fail. func (g *Gateway) UnregisterRPC(name string) { g.mu.Lock() defer g.mu.Unlock() if _, ok := g.handlers[handlerName(name)]; !ok { build.Critical("RPC not registered: " + name) } delete(g.handlers, handlerName(name)) } // RegisterConnectCall registers a name and RPCFunc to be called on a peer // upon connecting. func (g *Gateway) RegisterConnectCall(name string, fn modules.RPCFunc) { g.mu.Lock() defer g.mu.Unlock() if _, ok := g.initRPCs[name]; ok { build.Critical("ConnectCall already registered: " + name) } g.initRPCs[name] = fn } // UnregisterConnectCall unregisters an on-connect call and removes the // corresponding RPCFunc from g.initRPCs. Future connections to peers will not // trigger the RPC to be called on them. func (g *Gateway) UnregisterConnectCall(name string) { g.mu.Lock() defer g.mu.Unlock() if _, ok := g.initRPCs[name]; !ok { build.Critical("ConnectCall not registered: " + name) } delete(g.initRPCs, name) } // threadedListenPeer listens for new streams on a peer connection and serves them via // threadedHandleConn. func (g *Gateway) threadedListenPeer(p *peer) { // threadedListenPeer registers to the peerTG instead of the primary thread // group because peer connections can be lifetime in length, but can also // be short-lived. The fact that they can be lifetime means that they can't // call threads.Add as they will block calls to threads.Flush. The fact // that they can be short-lived means that threads.OnStop is not a good // tool for closing out the threads. Instead, they register to peerTG, // which is cleanly closed upon gateway shutdown but will not block any // calls to threads.Flush() if g.peerTG.Add() != nil { return } defer g.peerTG.Done() // Spin up a goroutine to listen for a shutdown signal from both the peer // and from the gateway. In the event of either, close the muxado session. connClosedChan := make(chan struct{}) peerCloseChan := make(chan struct{}) go func() { // Signal that the muxado session has been successfully closed, and // that this goroutine has terminated. defer close(connClosedChan) // Listen for a stop signal. select { case <-g.threads.StopChan(): case <-peerCloseChan: } // Can't call Disconnect because it could return sync.ErrStopped. g.mu.Lock() delete(g.peers, p.NetAddress) g.mu.Unlock() if err := p.sess.Close(); err != nil { g.log.Debugf("WARN: error disconnecting from peer %q: %v", p.NetAddress, err) } }() for { conn, err := p.accept() if err != nil { g.log.Debugf("Peer connection with %v closed: %v\n", p.NetAddress, err) break } // it is the handler's responsibility to close the connection go g.threadedHandleConn(conn) } // Signal that the goroutine can shutdown. close(peerCloseChan) // Wait for confirmation that the goroutine has shut down before returning // and releasing the threadgroup registration. <-connClosedChan } // threadedHandleConn reads header data from a connection, then routes it to the // appropriate handler for further processing. func (g *Gateway) threadedHandleConn(conn modules.PeerConn) { defer conn.Close() if g.threads.Add() != nil { return } defer g.threads.Done() var id rpcID if err := encoding.ReadObject(conn, &id, 8); err != nil { return } // call registered handler for this ID g.mu.RLock() fn, ok := g.handlers[id] g.mu.RUnlock() if !ok { g.log.Debugf("WARN: incoming conn %v requested unknown RPC \"%v\"", conn.RPCAddr(), id) return } g.log.Debugf("INFO: incoming conn %v requested RPC \"%v\"", conn.RPCAddr(), id) // call fn err := fn(conn) // don't log benign errors if err == modules.ErrDuplicateTransactionSet || err == modules.ErrBlockKnown { err = nil } if err != nil { g.log.Debugf("WARN: incoming RPC \"%v\" from conn %v failed: %v", id, conn.RPCAddr(), err) } } // Broadcast calls an RPC on all of the specified peers. The calls are run in // parallel. Broadcasts are restricted to "one-way" RPCs, which simply write an // object and disconnect. This is why Broadcast takes an interface{} instead of // an RPCFunc. func (g *Gateway) Broadcast(name string, obj interface{}, peers []modules.Peer) { if g.threads.Add() != nil { return } defer g.threads.Done() g.log.Debugf("INFO: broadcasting RPC %q to %v peers", name, len(peers)) // only encode obj once, instead of using WriteObject enc := encoding.Marshal(obj) fn := func(conn modules.PeerConn) error { return encoding.WritePrefix(conn, enc) } var wg sync.WaitGroup for _, p := range peers { wg.Add(1) go func(addr modules.NetAddress) { defer wg.Done() err := g.managedRPC(addr, name, fn) if err != nil { g.log.Debugf("WARN: broadcasting RPC %q to peer %q failed (attempting again in 10 seconds): %v", name, addr, err) // try one more time before giving up select { case <-time.After(10 * time.Second): case <-g.threads.StopChan(): return } err := g.managedRPC(addr, name, fn) if err != nil { g.log.Debugf("WARN: broadcasting RPC %q to peer %q failed twice: %v", name, addr, err) } } }(p.NetAddress) } wg.Wait() } Sia-1.0.4/modules/gateway/rpc_test.go000066400000000000000000000323661302033070500175160ustar00rootroot00000000000000package gateway import ( "io" "testing" "time" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" ) func TestRPCID(t *testing.T) { cases := map[rpcID]string{ rpcID{}: " ", rpcID{'f', 'o', 'o'}: "foo ", rpcID{'f', 'o', 'o', 'b', 'a', 'r', 'b', 'a'}: "foobarba", } for id, s := range cases { if id.String() != s { t.Errorf("rpcID.String mismatch: expected %v, got %v", s, id.String()) } } } func TestHandlerName(t *testing.T) { cases := map[string]rpcID{ "": {}, "foo": {'f', 'o', 'o'}, "foobarbaz": {'f', 'o', 'o', 'b', 'a', 'r', 'b', 'a'}, } for s, id := range cases { if hid := handlerName(s); hid != id { t.Errorf("handlerName mismatch: expected %v, got %v", id, hid) } } } // TestRegisterRPC tests that registering the same RPC twice causes a panic. func TestRegisterRPC(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestRegisterRPC", t) defer g.Close() g.RegisterRPC("Foo", func(conn modules.PeerConn) error { return nil }) defer func() { if r := recover(); r == nil { t.Error("Registering the same RPC twice did not cause a panic") } }() g.RegisterRPC("Foo", func(conn modules.PeerConn) error { return nil }) } // TestUnregisterRPC tests that unregistering an RPC causes calls to it to // fail, and checks that unregistering a non-registered RPC causes a panic. func TestUnregisterRPC(t *testing.T) { if testing.Short() { t.SkipNow() } g1 := newTestingGateway("TestUnregisterRPC1", t) defer g1.Close() g2 := newTestingGateway("TestUnregisterRPC2", t) defer g2.Close() err := g2.Connect(g1.Address()) if err != nil { t.Fatal(err) } dummyFunc := func(conn modules.PeerConn) error { var str string return encoding.ReadObject(conn, &str, 11) } // Register RPC and check that calling it succeeds. g1.RegisterRPC("Foo", func(conn modules.PeerConn) error { return encoding.WriteObject(conn, "foo") }) err = g2.RPC(g1.Address(), "Foo", dummyFunc) if err != nil { t.Errorf("calling registered RPC on g1 returned %q", err) } // Unregister RPC and check that calling it fails. g1.UnregisterRPC("Foo") err = g2.RPC(g1.Address(), "Foo", dummyFunc) if err != io.EOF { t.Errorf("calling unregistered RPC on g1 returned %q instead of io.EOF", err) } // Unregister again and check that it panics. defer func() { if r := recover(); r == nil { t.Error("Unregistering an unregistered RPC did not cause a panic") } }() g1.UnregisterRPC("Foo") } // TestRegisterConnectCall tests that registering the same on-connect call // twice causes a panic. func TestRegisterConnectCall(t *testing.T) { if testing.Short() { t.SkipNow() } g := newTestingGateway("TestRegisterConnectCall", t) defer g.Close() // Register an on-connect call. g.RegisterConnectCall("Foo", func(conn modules.PeerConn) error { return nil }) defer func() { if r := recover(); r == nil { t.Error("Registering the same on-connect call twice did not cause a panic") } }() g.RegisterConnectCall("Foo", func(conn modules.PeerConn) error { return nil }) } // TestUnregisterConnectCallPanics tests that unregistering the same on-connect // call twice causes a panic. func TestUnregisterConnectCallPanics(t *testing.T) { if testing.Short() { t.SkipNow() } g1 := newTestingGateway("TestUnregisterConnectCall1", t) defer g1.Close() g2 := newTestingGateway("TestUnregisterConnectCall2", t) defer g2.Close() rpcChan := make(chan struct{}) // Register on-connect call and test that RPC is called on connect. g1.RegisterConnectCall("Foo", func(conn modules.PeerConn) error { rpcChan <- struct{}{} return nil }) err := g1.Connect(g2.Address()) if err != nil { t.Fatal(err) } select { case <-rpcChan: case <-time.After(200 * time.Millisecond): t.Fatal("ConnectCall not called on Connect after it was registered") } // Disconnect, unregister on-connect call, and test that RPC is not called on connect. err = g1.Disconnect(g2.Address()) if err != nil { t.Fatal(err) } g1.UnregisterConnectCall("Foo") err = g1.Connect(g2.Address()) if err != nil { t.Fatal(err) } select { case <-rpcChan: t.Fatal("ConnectCall called on Connect after it was unregistered") case <-time.After(200 * time.Millisecond): } // Unregister again and check that it panics. defer func() { if r := recover(); r == nil { t.Error("Unregistering an unregistered on-connect call did not cause a panic") } }() g1.UnregisterConnectCall("Foo") } func TestRPC(t *testing.T) { if testing.Short() { t.SkipNow() } g1 := newTestingGateway("TestRPC1", t) defer g1.Close() if err := g1.RPC("foo.com:123", "", nil); err == nil { t.Fatal("RPC on unconnected peer succeeded") } g2 := newTestingGateway("TestRPC2", t) defer g2.Close() err := g1.Connect(g2.Address()) if err != nil { t.Fatal("failed to connect:", err) } g2.RegisterRPC("Foo", func(conn modules.PeerConn) error { var i uint64 err := encoding.ReadObject(conn, &i, 8) if err != nil { return err } else if i == 0xdeadbeef { return encoding.WriteObject(conn, "foo") } else { return encoding.WriteObject(conn, "bar") } }) var foo string err = g1.RPC(g2.Address(), "Foo", func(conn modules.PeerConn) error { err := encoding.WriteObject(conn, 0xdeadbeef) if err != nil { return err } return encoding.ReadObject(conn, &foo, 11) }) if err != nil { t.Fatal(err) } if foo != "foo" { t.Fatal("Foo gave wrong response:", foo) } // wrong number should produce an error err = g1.RPC(g2.Address(), "Foo", func(conn modules.PeerConn) error { err := encoding.WriteObject(conn, 0xbadbeef) if err != nil { return err } return encoding.ReadObject(conn, &foo, 11) }) if err != nil { t.Fatal(err) } if foo != "bar" { t.Fatal("Foo gave wrong response:", foo) } // don't read or write anything err = g1.RPC(g2.Address(), "Foo", func(modules.PeerConn) error { return errNoPeers // any non-nil error will do }) if err == nil { t.Fatal("bad RPC did not produce an error") } g1.peers[g2.Address()].sess.Close() if err := g1.RPC(g2.Address(), "Foo", nil); err == nil { t.Fatal("RPC on closed peer connection succeeded") } } func TestThreadedHandleConn(t *testing.T) { if testing.Short() { t.SkipNow() } g1 := newTestingGateway("TestThreadedHandleConn1", t) defer g1.Close() g2 := newTestingGateway("TestThreadedHandleConn2", t) defer g2.Close() err := g1.Connect(g2.Address()) if err != nil { t.Fatal("failed to connect:", err) } g2.RegisterRPC("Foo", func(conn modules.PeerConn) error { var i uint64 err := encoding.ReadObject(conn, &i, 8) if err != nil { return err } else if i == 0xdeadbeef { return encoding.WriteObject(conn, "foo") } else { return encoding.WriteObject(conn, "bar") } }) // custom rpc fn (doesn't automatically write rpcID) rpcFn := func(fn func(modules.PeerConn) error) error { conn, err := g1.peers[g2.Address()].open() if err != nil { return err } defer conn.Close() return fn(conn) } // bad rpcID err = rpcFn(func(conn modules.PeerConn) error { return encoding.WriteObject(conn, [3]byte{1, 2, 3}) }) if err != nil { t.Fatal("rpcFn failed:", err) } // unknown rpcID err = rpcFn(func(conn modules.PeerConn) error { return encoding.WriteObject(conn, handlerName("bar")) }) if err != nil { t.Fatal("rpcFn failed:", err) } // valid rpcID err = rpcFn(func(conn modules.PeerConn) error { return encoding.WriteObject(conn, handlerName("Foo")) }) if err != nil { t.Fatal("rpcFn failed:", err) } } // TestBroadcast tests that calling broadcast with a slice of peers only // broadcasts to those peers. func TestBroadcast(t *testing.T) { if testing.Short() { t.SkipNow() } g1 := newTestingGateway("TestBroadcast1", t) defer g1.Close() g2 := newTestingGateway("TestBroadcast2", t) defer g2.Close() g3 := newTestingGateway("TestBroadcast3", t) defer g3.Close() err := g1.Connect(g2.Address()) if err != nil { t.Fatal("failed to connect:", err) } err = g1.Connect(g3.Address()) if err != nil { t.Fatal("failed to connect:", err) } var g2Payload, g3Payload string g2DoneChan := make(chan struct{}) g3DoneChan := make(chan struct{}) bothDoneChan := make(chan struct{}) g2.RegisterRPC("Recv", func(conn modules.PeerConn) error { encoding.ReadObject(conn, &g2Payload, 100) g2DoneChan <- struct{}{} return nil }) g3.RegisterRPC("Recv", func(conn modules.PeerConn) error { encoding.ReadObject(conn, &g3Payload, 100) g3DoneChan <- struct{}{} return nil }) // Test that broadcasting to all peers in g1.Peers() broadcasts to all peers. peers := g1.Peers() g1.Broadcast("Recv", "bar", peers) go func() { <-g2DoneChan <-g3DoneChan bothDoneChan <- struct{}{} }() select { case <-bothDoneChan: // Both g2 and g3 should receive the broadcast. case <-time.After(200 * time.Millisecond): t.Fatal("broadcasting to gateway.Peers() should broadcast to all peers") } if g2Payload != "bar" || g3Payload != "bar" { t.Fatal("broadcast failed:", g2Payload, g3Payload) } // Test that broadcasting to only g2 does not broadcast to g3. peers = make([]modules.Peer, 0) for _, p := range g1.Peers() { if p.NetAddress == g2.Address() { peers = append(peers, p) break } } g1.Broadcast("Recv", "baz", peers) select { case <-g2DoneChan: // Only g2 should receive a broadcast. case <-g3DoneChan: t.Error("broadcast broadcasted to peers not in the peers arg") case <-time.After(200 * time.Millisecond): t.Fatal("called broadcast with g2 in peers list, but g2 didn't receive it.") } if g2Payload != "baz" { t.Fatal("broadcast failed:", g2Payload) } // Test that broadcasting to only g3 does not broadcast to g2. peers = make([]modules.Peer, 0) for _, p := range g1.Peers() { if p.NetAddress == g3.Address() { peers = append(peers, p) break } } g1.Broadcast("Recv", "qux", peers) select { case <-g2DoneChan: t.Error("broadcast broadcasted to peers not in the peers arg") case <-g3DoneChan: // Only g3 should receive a broadcast. case <-time.After(200 * time.Millisecond): t.Fatal("called broadcast with g3 in peers list, but g3 didn't receive it.") } if g3Payload != "qux" { t.Fatal("broadcast failed:", g3Payload) } // Test that broadcasting to an empty slice (but not nil!) does not broadcast // to g2 or g3. peers = make([]modules.Peer, 0) g1.Broadcast("Recv", "quux", peers) select { case <-g2DoneChan: t.Error("broadcast broadcasted to peers not in the peers arg") case <-g3DoneChan: t.Error("broadcast broadcasted to peers not in the peers arg") case <-time.After(200 * time.Millisecond): // Neither peer should receive a broadcast. } // Test that calling broadcast with nil peers does not broadcast to g2 or g3. g1.Broadcast("Recv", "foo", nil) select { case <-g2DoneChan: t.Error("broadcast broadcasted to peers not in the peers arg") case <-g3DoneChan: t.Error("broadcast broadcasted to peers not in the peers arg") case <-time.After(200 * time.Millisecond): // Neither peer should receive a broadcast. } } // TestOutboundAndInboundRPCs tests that both inbound and outbound connections // can successfully make RPC calls. func TestOutboundAndInboundRPCs(t *testing.T) { if testing.Short() { t.SkipNow() } g1 := newTestingGateway("TestRPC1", t) defer g1.Close() g2 := newTestingGateway("TestRPC2", t) defer g2.Close() rpcChanG1 := make(chan struct{}) rpcChanG2 := make(chan struct{}) g1.RegisterRPC("recv", func(conn modules.PeerConn) error { rpcChanG1 <- struct{}{} return nil }) g2.RegisterRPC("recv", func(conn modules.PeerConn) error { rpcChanG2 <- struct{}{} return nil }) err := g1.Connect(g2.Address()) if err != nil { t.Fatal(err) } time.Sleep(10 * time.Millisecond) err = g1.RPC(g2.Address(), "recv", func(conn modules.PeerConn) error { return nil }) if err != nil { t.Fatal(err) } <-rpcChanG2 // Call the "recv" RPC on g1. We don't know g1's address as g2 sees it, so we // get it from the first address in g2's peer list. var addr modules.NetAddress for p_addr := range g2.peers { addr = p_addr break } err = g2.RPC(addr, "recv", func(conn modules.PeerConn) error { return nil }) if err != nil { t.Fatal(err) } <-rpcChanG1 } // TestCallingRPCFromRPC tests that calling an RPC from an RPC works. func TestCallingRPCFromRPC(t *testing.T) { if testing.Short() { t.SkipNow() } g1 := newTestingGateway("TestCallingRPCFromRPC1", t) defer g1.Close() g2 := newTestingGateway("TestCallingRPCFromRPC2", t) defer g2.Close() errChan := make(chan error) g1.RegisterRPC("FOO", func(conn modules.PeerConn) error { err := g1.RPC(conn.RPCAddr(), "BAR", func(conn modules.PeerConn) error { return nil }) errChan <- err return err }) barChan := make(chan struct{}) g2.RegisterRPC("BAR", func(conn modules.PeerConn) error { barChan <- struct{}{} return nil }) err := g1.Connect(g2.Address()) if err != nil { t.Fatal(err) } // Wait for g2 to accept the connection for { if len(g2.Peers()) > 0 { break } } err = g2.RPC(g1.Address(), "FOO", func(conn modules.PeerConn) error { return nil }) select { case err = <-errChan: if err != nil { t.Fatal(err) } case <-time.After(500 * time.Millisecond): t.Fatal("expected FOO RPC to be called") } select { case <-barChan: case <-time.After(500 * time.Millisecond): t.Fatal("expected BAR RPC to be called") } } Sia-1.0.4/modules/gateway/upnp.go000066400000000000000000000061121302033070500166430ustar00rootroot00000000000000package gateway import ( "errors" "io" "io/ioutil" "net" "net/http" "strconv" "strings" "time" "github.com/NebulousLabs/go-upnp" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" ) // myExternalIP discovers the gateway's external IP by querying a centralized // service, http://myexternalip.com. func myExternalIP() (string, error) { // timeout after 10 seconds client := http.Client{Timeout: time.Duration(10 * time.Second)} resp, err := client.Get("http://myexternalip.com/raw") if err != nil { return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { errResp, _ := ioutil.ReadAll(resp.Body) return "", errors.New(string(errResp)) } buf, err := ioutil.ReadAll(io.LimitReader(resp.Body, 64)) if err != nil { return "", err } if len(buf) == 0 { return "", errors.New("myexternalip.com returned a 0 length IP address") } // trim newline return strings.TrimSpace(string(buf)), nil } // threadedLearnHostname discovers the external IP of the Gateway. Once the IP // has been discovered, it registers the ShareNodes RPC to be called on new // connections, advertising the IP to other nodes. func (g *Gateway) threadedLearnHostname() { if err := g.threads.Add(); err != nil { return } defer g.threads.Done() if build.Release == "testing" { return } // try UPnP first, then fallback to myexternalip.com var host string d, err := upnp.Discover() if err == nil { host, err = d.ExternalIP() } if err != nil { host, err = myExternalIP() } if err != nil { g.log.Println("WARN: failed to discover external IP:", err) return } g.mu.RLock() addr := modules.NetAddress(net.JoinHostPort(host, g.port)) g.mu.RUnlock() if err := addr.IsValid(); err != nil { g.log.Printf("WARN: discovered hostname %q is invalid: %v", addr, err) return } g.mu.Lock() g.myAddr = addr g.mu.Unlock() g.log.Println("INFO: our address is", addr) } // threadedForwardPort adds a port mapping to the router. func (g *Gateway) threadedForwardPort(port string) { if err := g.threads.Add(); err != nil { return } defer g.threads.Done() if build.Release == "testing" { return } d, err := upnp.Discover() if err != nil { g.log.Printf("WARN: could not automatically forward port %s: no UPnP-enabled devices found: %v", port, err) return } portInt, _ := strconv.Atoi(port) err = d.Forward(uint16(portInt), "Sia RPC") if err != nil { g.log.Printf("WARN: could not automatically forward port %s: %v", port, err) return } g.log.Println("INFO: successfully forwarded port", port) // Establish port-clearing at shutdown. g.threads.AfterStop(func() { g.managedClearPort(port) }) } // managedClearPort removes a port mapping from the router. func (g *Gateway) managedClearPort(port string) { if build.Release == "testing" { return } d, err := upnp.Discover() if err != nil { return } portInt, _ := strconv.Atoi(port) err = d.Clear(uint16(portInt)) if err != nil { g.log.Printf("WARN: could not automatically unforward port %s: %v", port, err) return } g.log.Println("INFO: successfully unforwarded port", port) } Sia-1.0.4/modules/host.go000066400000000000000000000115311302033070500151760ustar00rootroot00000000000000package modules import ( "github.com/NebulousLabs/Sia/types" ) const ( // HostDir names the directory that contains the host persistence. HostDir = "host" ) var ( // BytesPerTerabyte is the conversion rate between bytes and terabytes. BytesPerTerabyte = types.NewCurrency64(1e12) // BlockBytesPerMonthTerabyte is the conversion rate between block-bytes and month-TB. BlockBytesPerMonthTerabyte = BytesPerTerabyte.Mul64(4320) ) type ( // HostFinancialMetrics provides financial statistics for the host, // including money that is locked in contracts. Though verbose, these // statistics should provide a clear picture of where the host's money is // currently being used. The front end can consolidate stats where desired. // Potential revenue refers to revenue that is available in a file // contract for which the file contract window has not yet closed. HostFinancialMetrics struct { // Every time a renter forms a contract with a host, a contract fee is // paid by the renter. These stats track the total contract fees. ContractCount uint64 `json:"contractcount"` ContractCompensation types.Currency `json:"contractcompensation"` PotentialContractCompensation types.Currency `json:"potentialcontractcompensation"` // Metrics related to storage proofs, collateral, and submitting // transactions to the blockchain. LockedStorageCollateral types.Currency `json:"lockedstoragecollateral"` LostRevenue types.Currency `json:"lostrevenue"` LostStorageCollateral types.Currency `json:"loststoragecollateral"` PotentialStorageRevenue types.Currency `json:"potentialstoragerevenue"` RiskedStorageCollateral types.Currency `json:"riskedstoragecollateral"` StorageRevenue types.Currency `json:"storagerevenue"` TransactionFeeExpenses types.Currency `json:"transactionfeeexpenses"` // Bandwidth financial metrics. DownloadBandwidthRevenue types.Currency `json:"downloadbandwidthrevenue"` PotentialDownloadBandwidthRevenue types.Currency `json:"potentialdownloadbandwidthrevenue"` PotentialUploadBandwidthRevenue types.Currency `json:"potentialuploadbandwidthrevenue"` UploadBandwidthRevenue types.Currency `json:"uploadbandwidthrevenue"` } // HostInternalSettings contains a list of settings that can be changed. HostInternalSettings struct { AcceptingContracts bool `json:"acceptingcontracts"` MaxDownloadBatchSize uint64 `json:"maxdownloadbatchsize"` MaxDuration types.BlockHeight `json:"maxduration"` MaxReviseBatchSize uint64 `json:"maxrevisebatchsize"` NetAddress NetAddress `json:"netaddress"` WindowSize types.BlockHeight `json:"windowsize"` Collateral types.Currency `json:"collateral"` CollateralBudget types.Currency `json:"collateralbudget"` MaxCollateral types.Currency `json:"maxcollateral"` MinContractPrice types.Currency `json:"mincontractprice"` MinDownloadBandwidthPrice types.Currency `json:"mindownloadbandwidthprice"` MinStoragePrice types.Currency `json:"minstorageprice"` MinUploadBandwidthPrice types.Currency `json:"minuploadbandwidthprice"` } // HostNetworkMetrics reports the quantity of each type of RPC call that // has been made to the host. HostNetworkMetrics struct { DownloadCalls uint64 `json:"downloadcalls"` ErrorCalls uint64 `json:"errorcalls"` FormContractCalls uint64 `json:"formcontractcalls"` RenewCalls uint64 `json:"renewcalls"` ReviseCalls uint64 `json:"revisecalls"` SettingsCalls uint64 `json:"settingscalls"` UnrecognizedCalls uint64 `json:"unrecognizedcalls"` } // A Host can take storage from disk and offer it to the network, managing // things such as announcements, settings, and implementing all of the RPCs // of the host protocol. Host interface { // Announce submits a host announcement to the blockchain. Announce() error // AnnounceAddress submits an announcement using the given address. AnnounceAddress(NetAddress) error // ExternalSettings returns the settings of the host as seen by an // untrusted node querying the host for settings. ExternalSettings() HostExternalSettings // FinancialMetrics returns the financial statistics of the host. FinancialMetrics() HostFinancialMetrics // InternalSettings returns the host's internal settings, including // potentially private or sensitive information. InternalSettings() HostInternalSettings // NetworkMetrics returns information on the types of RPC calls that // have been made to the host. NetworkMetrics() HostNetworkMetrics // SetInternalSettings sets the hosting parameters of the host. SetInternalSettings(HostInternalSettings) error // The storage manager provides an interface for adding and removing // storage folders and data sectors to the host. StorageManager } ) Sia-1.0.4/modules/host/000077500000000000000000000000001302033070500146465ustar00rootroot00000000000000Sia-1.0.4/modules/host/announce.go000066400000000000000000000053231302033070500170060ustar00rootroot00000000000000package host import ( "errors" "github.com/NebulousLabs/Sia/modules" ) var ( // errAnnWalletLocked is returned during a host announcement if the wallet // is locked. errAnnWalletLocked = errors.New("cannot announce the host while the wallet is locked") // errUnknownAddress is returned if the host is unable to determine a // public address for itself to use in the announcement. errUnknownAddress = errors.New("host cannot announce, does not seem to have a valid address.") ) // announce creates an announcement transaction and submits it to the network. func (h *Host) announce(addr modules.NetAddress) error { // The wallet needs to be unlocked to add fees to the transaction, and the // host needs to have an active unlock hash that renters can make payment // to. if !h.wallet.Unlocked() { return errAnnWalletLocked } err := h.checkUnlockHash() if err != nil { return err } // Create the announcement that's going to be added to the arbitrary data // field of the transaction. signedAnnouncement, err := modules.CreateAnnouncement(addr, h.publicKey, h.secretKey) if err != nil { return err } // Create a transaction, with a fee, that contains the full announcement. txnBuilder := h.wallet.StartTransaction() _, fee := h.tpool.FeeEstimation() fee = fee.Mul64(500) // Estimated txn size (in bytes) of a host announcement. err = txnBuilder.FundSiacoins(fee) if err != nil { txnBuilder.Drop() return err } _ = txnBuilder.AddMinerFee(fee) _ = txnBuilder.AddArbitraryData(signedAnnouncement) txnSet, err := txnBuilder.Sign(true) if err != nil { txnBuilder.Drop() return err } // Add the transactions to the transaction pool. err = h.tpool.AcceptTransactionSet(txnSet) if err != nil { txnBuilder.Drop() return err } h.announced = true h.log.Printf("INFO: Successfully announced as %v", addr) return nil } // Announce creates a host announcement transaction, adding information to the // arbitrary data, signing the transaction, and submitting it to the // transaction pool. func (h *Host) Announce() error { h.mu.Lock() defer h.mu.Unlock() err := h.tg.Add() if err != nil { return err } defer h.tg.Done() // Determine whether to use the settings.NetAddress or autoAddress. if h.settings.NetAddress != "" { return h.announce(h.settings.NetAddress) } if h.autoAddress == "" { return errUnknownAddress } return h.announce(h.autoAddress) } // AnnounceAddress submits a host announcement to the blockchain to announce a // specific address. No checks for validity are performed on the address. func (h *Host) AnnounceAddress(addr modules.NetAddress) error { h.mu.Lock() defer h.mu.Unlock() err := h.tg.Add() if err != nil { return err } defer h.tg.Done() return h.announce(addr) } Sia-1.0.4/modules/host/announce_test.go000066400000000000000000000067241302033070500200530ustar00rootroot00000000000000package host import ( "bytes" "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // announcementFinder is a quick module that parses the blockchain for host // announcements, keeping a record of all the announcements that get found. type announcementFinder struct { cs modules.ConsensusSet // Announcements that have been seen. The two slices are wedded. netAddresses []modules.NetAddress publicKeys []types.SiaPublicKey } // ProcessConsensusChange receives consensus changes from the consensus set and // parses them for valid host announcements. func (af *announcementFinder) ProcessConsensusChange(cc modules.ConsensusChange) { for _, block := range cc.AppliedBlocks { for _, txn := range block.Transactions { for _, arb := range txn.ArbitraryData { addr, pubKey, err := modules.DecodeAnnouncement(arb) if err == nil { af.netAddresses = append(af.netAddresses, addr) af.publicKeys = append(af.publicKeys, pubKey) } } } } } // Close will shut down the announcement finder. func (af *announcementFinder) Close() error { af.cs.Unsubscribe(af) return nil } // newAnnouncementFinder will create and return an announcement finder. func newAnnouncementFinder(cs modules.ConsensusSet) (*announcementFinder, error) { af := &announcementFinder{ cs: cs, } err := cs.ConsensusSetSubscribe(af, modules.ConsensusChangeBeginning) if err != nil { return nil, err } return af, nil } // TestHostAnnounce checks that the host announce function is operating // correctly. func TestHostAnnounce(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := newHostTester("TestHostAnnounce") if err != nil { t.Fatal(err) } defer ht.Close() // Create an announcement finder to scan the blockchain for host // announcements. af, err := newAnnouncementFinder(ht.cs) if err != nil { t.Fatal(err) } defer af.Close() // Create an announcement, then use the address finding module to scan the // blockchain for the host's address. err = ht.host.Announce() if err != nil { t.Fatal(err) } _, err = ht.miner.AddBlock() if err != nil { t.Fatal(err) } if len(af.publicKeys) != 1 { t.Fatal("could not find host announcement in blockchain") } if af.netAddresses[0] != ht.host.autoAddress { t.Error("announcement has wrong address") } if !bytes.Equal(af.publicKeys[0].Key, ht.host.publicKey.Key) { t.Error("announcement has wrong host key") } } // TestHostAnnounceAddress checks that the host announce address function is // operating correctly. func TestHostAnnounceAddress(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := newHostTester("TestHostAnnounceAddress") if err != nil { t.Fatal(err) } defer ht.Close() // Create an announcement finder to scan the blockchain for host // announcements. af, err := newAnnouncementFinder(ht.cs) if err != nil { t.Fatal(err) } defer af.Close() // Create an announcement, then use the address finding module to scan the // blockchain for the host's address. addr := modules.NetAddress("foo.com:1234") err = ht.host.AnnounceAddress(addr) if err != nil { t.Fatal(err) } _, err = ht.miner.AddBlock() if err != nil { t.Fatal(err) } if len(af.netAddresses) != 1 { t.Fatal("could not find host announcement in blockchain") } if af.netAddresses[0] != addr { t.Error("announcement has wrong address") } if !bytes.Equal(af.publicKeys[0].Key, ht.host.publicKey.Key) { t.Error("announcement has wrong host key") } } Sia-1.0.4/modules/host/consts.go000066400000000000000000000240751302033070500165160ustar00rootroot00000000000000package host import ( "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "time" ) const ( // defaultMaxDuration defines the maximum number of blocks into the future // that the host will accept for the duration of an incoming file contract // obligation. 6 months is chosen because hosts are expected to be // long-term entities, and because we want to have a set of hosts that // support 6 month contracts when Sia leaves beta. defaultMaxDuration = 144 * 30 * 6 // 6 months. // fileContractNegotiationTimeout indicates the amount of time that a // renter has to negotiate a file contract with the host. A timeout is // necessary to limit the impact of DoS attacks. fileContractNegotiationTimeout = 120 * time.Second // iteratedConnectionTime is the amount of time that is allowed to pass // before the host will stop accepting new iterations on an iterated // connection. iteratedConnectionTime = 1200 * time.Second // resubmissionTimeout defines the number of blocks that a host will wait // before attempting to resubmit a transaction to the blockchain. // Typically, this transaction will contain either a file contract, a file // contract revision, or a storage proof. resubmissionTimeout = 3 ) var ( // defaultCollateral defines the amount of money that the host puts up as // collateral per-byte by default. The collateral should be considered as // an absolute instead of as a percentage, because low prices result in // collaterals which may be significant by percentage, but insignificant // overall. A default of 25 KS / TB / Month has been chosen, which is 2.5x // the default price for storage. The host is expected to put up a // significant amount of collateral as a commitment to faithfulness, // because this guarantees that the incentives are aligned for the host to // keep the data even if the price of siacoin fluctuates, the price of raw // storage fluctuates, or the host realizes that there is unexpected // opportunity cost in being a host. defaultCollateral = types.SiacoinPrecision.Mul64(500).Div(modules.BlockBytesPerMonthTerabyte) // 500 SC / TB / Month // defaultCollateralBudget defines the maximum number of siacoins that the // host is going to allocate towards collateral. The number has been chosen // as a number that is large, but not so large that someone would be // furious for losing access to it for a few weeks. defaultCollateralBudget = types.SiacoinPrecision.Mul64(2e6) // defaultContractPrice defines the default price of creating a contract // with the host. The default is set to 30 siacoins, which the file // contract revision can have 15 siacoins put towards it, and the storage // proof can have 15 siacoins put towards it. defaultContractPrice = types.SiacoinPrecision.Mul64(20) // 20 siacoins // defaultDownloadBandwidthPrice defines the default price of upload // bandwidth. The default is set to 10 siacoins per gigabyte, because // download bandwidth is expected to be plentiful but also in-demand. defaultDownloadBandwidthPrice = types.SiacoinPrecision.Mul64(250).Div(modules.BytesPerTerabyte) // 250 SC / TB // defaultMaxDownloadBatchSize defines the maximum number of bytes that the // host will allow to be requested by a single download request. 17 MiB has // been chosen because it's 4 full sectors plus some wiggle room. 17 MiB is // a conservative default, most hosts will be fine with a number like 65 // MiB. defaultMaxDownloadBatchSize = 17 * (1 << 20) // defaultMaxReviseBatchSize defines the maximum number of bytes that the // host will allow to be sent during a single batch update in a revision // RPC. 17 MiB has been chosen because it's four full sectors, plus some // wiggle room for the extra data or a few delete operations. The whole // batch will be held in memory, so the batch size should only be increased // substantially if the host has a lot of memory. Additionally, the whole // batch is sent in one network connection. Additionally, the renter can // steal funds for upload bandwidth all the way out to the size of a batch. // 17 MiB is a conservative default, most hosts are likely to be just fine // with a number like 65 MiB. defaultMaxReviseBatchSize = 17 * (1 << 20) // defaultMaxCollateral defines the maximum amount of collateral that the // host is comfortable putting into a single file contract. 10e3 is a // relatively small file contract, but millions of siacoins could be locked // away by only a few hundred file contracts. As the ecosystem matures, it // is expected that the safe default for this value will increase quite a // bit. defaultMaxCollateral = types.SiacoinPrecision.Mul64(100e3) // defaultStoragePrice defines the starting price for hosts selling // storage. We try to match a number that is both reasonably profitable and // reasonably competitive. defaultStoragePrice = types.SiacoinPrecision.Mul64(500).Div(modules.BlockBytesPerMonthTerabyte) // 500 SC / TB / Month // defaultUploadBandwidthPrice defines the default price of upload // bandwidth. The default is set to 1 siacoin per GB, because the host is // presumed to have a large amount of downstream bandwidth. Furthermore, // the host is typically only downloading data if it is planning to store // the data, meaning that the host serves to profit from accepting the // data. defaultUploadBandwidthPrice = types.SiacoinPrecision.Mul64(10).Div(modules.BytesPerTerabyte) // 10 SC / TB // defaultWindowSize is the size of the proof of storage window requested // by the host. The host will not delete any obligations until the window // has closed and buried under several confirmations. For release builds, // the default is set to 144 blocks, or about 1 day. This gives the host // flexibility to experience downtime without losing file contracts. The // optimal default, especially as the network matures, is probably closer // to 36 blocks. An experienced or high powered host should not be // frustrated by lost coins due to long periods of downtime. defaultWindowSize = func() types.BlockHeight { if build.Release == "dev" { return 36 // 3.6 minutes. } if build.Release == "standard" { return 144 // 1 day. } if build.Release == "testing" { return 5 // 5 seconds. } panic("unrecognized release constant in host - defaultWindowSize") }() // logAllLimit is the number of errors of each type that the host will log // before switching to probabilistic logging. If there are not many errors, // it is reasonable that all errors get logged. If there are lots of // errors, to cut down on the noise only some of the errors get logged. logAllLimit = func() uint64 { if build.Release == "dev" { return 50 } if build.Release == "standard" { return 250 } if build.Release == "testing" { return 100 } panic("unrecognized release constant in host - logAllLimit") }() // logFewLimit is the number of errors of each type that the host will log // before substantially constricting the amount of logging that it is // doing. logFewLimit = func() uint64 { if build.Release == "dev" { return 500 } if build.Release == "standard" { return 2500 } if build.Release == "testing" { return 500 } panic("unrecognized release constant in host - logAllLimit") }() // maximumLockedStorageObligations sets the maximum number of storage // obligations that are allowed to be locked at a time. The map uses an // in-memory lock, but also a locked storage obligation could be reading a // whole sector into memory, which could use a bunch of system resources. maximumLockedStorageObligations = func() uint64 { if build.Release == "dev" { return 20 } if build.Release == "standard" { return 100 } if build.Release == "testing" { return 5 } panic("unrecognized release constant in host - maximumLockedStorageObligations") }() // obligationLockTimeout defines how long a thread will wait to get a lock // on a storage obligation before timing out and reporting an error to the // renter. obligationLockTimeout = func() time.Duration { if build.Release == "dev" { return time.Second * 20 } if build.Release == "standard" { return time.Second * 60 } if build.Release == "testing" { return time.Second * 3 } panic("unrecognized release constant in host - obligationLockTimeout") }() // revisionSubmissionBuffer describes the number of blocks ahead of time // that the host will submit a file contract revision. The host will not // accept any more revisions once inside the submission buffer. revisionSubmissionBuffer = func() types.BlockHeight { if build.Release == "dev" { return 20 // About 4 minutes } if build.Release == "standard" { return 144 // 1 day. } if build.Release == "testing" { return 4 } panic("unrecognized release constant in host - revision submission buffer") }() ) // All of the following variables define the names of buckets used by the host // in the database. var ( // bucketActionItems maps a blockchain height to a list of storage // obligations that need to be managed in some way at that height. The // height is stored as a big endian uint64, which means that bolt will // store the heights sorted in numerical order. The action item itself is // an array of file contract ids. The host is able to contextually figure // out what the necessary actions for that item are based on the file // contract id and the associated storage obligation that can be retrieved // using the id. bucketActionItems = []byte("BucketActionItems") // bucketStorageObligations contains a set of serialized // 'storageObligations' sorted by their file contract id. bucketStorageObligations = []byte("BucketStorageObligations") ) // init runs a series of sanity checks to verify that the constants have sane // values. func init() { // The revision submission buffer should be greater than the resubmission // timeout, because there should be time to perform resubmission if the // first attempt to submit the revision fails. if revisionSubmissionBuffer < resubmissionTimeout { build.Critical("revision submission buffer needs to be larger than or equal to the resubmission timeout") } } Sia-1.0.4/modules/host/dependencies.go000066400000000000000000000111771302033070500176320ustar00rootroot00000000000000package host import ( "crypto/rand" "errors" "io/ioutil" "net" "os" "github.com/NebulousLabs/Sia/persist" ) // Fake errors that get returned when a simulated failure of a dependency is // desired for testing. var ( mockErrListen = errors.New("simulated Listen failure") mockErrLoadFile = errors.New("simulated LoadFile failure") mockErrMkdirAll = errors.New("simulated MkdirAll failure") mockErrNewLogger = errors.New("simulated NewLogger failure") mockErrOpenDatabase = errors.New("simulated OpenDatabase failure") mockErrReadFile = errors.New("simulated ReadFile failure") mockErrRemoveFile = errors.New("simulated RemoveFile faulure") mockErrSymlink = errors.New("simulated Symlink failure") mockErrWriteFile = errors.New("simulated WriteFile failure") ) // These interfaces define the Host's dependencies. Mocking implementation // complexity can be reduced by defining each dependency as the minimum // possible subset of the real dependency. type ( // dependencies defines all of the dependencies of the Host. dependencies interface { // disrupt can be inserted in the code as a way to inject problems, // such as a network call that take 10 minutes or a disk write that // never completes. disrupt will return true if the disruption is // forcibly triggered. In production, disrupt will always return false. disrupt(string) bool // listen gives the host the ability to receive incoming connections. listen(string, string) (net.Listener, error) // loadFile allows the host to load a persistence structure form disk. loadFile(persist.Metadata, interface{}, string) error // mkdirAll gives the host the ability to create chains of folders // within the filesystem. mkdirAll(string, os.FileMode) error // newLogger creates a logger that the host can use to log messages and // write critical statements. newLogger(string) (*persist.Logger, error) // openDatabase creates a database that the host can use to interact // with large volumes of persistent data. openDatabase(persist.Metadata, string) (*persist.BoltDatabase, error) // randRead fills the input bytes with random data. randRead([]byte) (int, error) // readFile reads a file in full from the filesystem. readFile(string) ([]byte, error) // removeFile removes a file from file filesystem. removeFile(string) error // symlink creates a sym link between a source and a destination. symlink(s1, s2 string) error // writeFile writes data to the filesystem using the provided filename. writeFile(string, []byte, os.FileMode) error } ) type ( // productionDependencies is an empty struct that implements all of the // dependencies using full featured libraries. productionDependencies struct{} ) // disrupt will always return false, but can be over-written during testing to // trigger disruptions. func (productionDependencies) disrupt(string) bool { return false } // listen gives the host the ability to receive incoming connections. func (productionDependencies) listen(s1, s2 string) (net.Listener, error) { return net.Listen(s1, s2) } // loadFile allows the host to load a persistence structure form disk. func (productionDependencies) loadFile(m persist.Metadata, i interface{}, s string) error { return persist.LoadFile(m, i, s) } // mkdirAll gives the host the ability to create chains of folders within the // filesystem. func (productionDependencies) mkdirAll(s string, fm os.FileMode) error { return os.MkdirAll(s, fm) } // newLogger creates a logger that the host can use to log messages and write // critical statements. func (productionDependencies) newLogger(s string) (*persist.Logger, error) { return persist.NewFileLogger(s) } // openDatabase creates a database that the host can use to interact with large // volumes of persistent data. func (productionDependencies) openDatabase(m persist.Metadata, s string) (*persist.BoltDatabase, error) { return persist.OpenDatabase(m, s) } // randRead fills the input bytes with random data. func (productionDependencies) randRead(b []byte) (int, error) { return rand.Read(b) } // readFile reads a file from the filesystem. func (productionDependencies) readFile(s string) ([]byte, error) { return ioutil.ReadFile(s) } // removeFile removes a file from the filesystem. func (productionDependencies) removeFile(s string) error { return os.Remove(s) } // symlink creates a symlink between a source and a destination file. func (productionDependencies) symlink(s1, s2 string) error { return os.Symlink(s1, s2) } // writeFile writes a file to the filesystem. func (productionDependencies) writeFile(s string, b []byte, fm os.FileMode) error { return ioutil.WriteFile(s, b, fm) } Sia-1.0.4/modules/host/errors.go000066400000000000000000000130321302033070500165100ustar00rootroot00000000000000package host // errors.go is responsible for logging the various errors that the host runs // into related to operations that cannot immediately provide feedback to the // user. (e.g. network failures, disk failures, etc.). Different errors should // be handled and logged differently, depending on severity and frequency, such // that the person reading the logs is able to see all of the major issues // without having them obstructed by the minor ones. import ( "errors" "strings" "sync/atomic" "github.com/NebulousLabs/Sia/crypto" ) const ( errorCommunicationProbability = 5 errorConnectionProbability = 20 errorConsensusProbability = 1 errorInternalProbability = 3 errorNormalProbability = 20 ) type ( // ErrorCommunication errors are meant to be returned if the host and the // renter seem to be miscommunicating. For example, if the renter attempts // to pay an insufficient price, there has been a communication error. ErrorCommunication string // ErrorConnection is meant to be used on errors where the network is // returning unexpected errors. For example, sudden disconnects or // connection write failures. ErrorConnection string // ErrorConsensus errors are meant to be used when there are problems // related to consensus, such as an inability to submit a storage proof to // the blockchain, or an inability to get a file contract revision on to // the blockchain. ErrorConsensus string // ErrorInternal errors are meant to be used if an internal process in the // host is malfunctioning, for example if the disk is failing. ErrorInternal string ) // composeErrors will take multiple errors and compose them into a single // errors with a longer message. Any nil errors used as inputs will be stripped // out, and if there are zero non-nil inputs then 'nil' will be returned. // // The original types of the errors is not preserved at all. func composeErrors(errs ...error) error { // Strip out any nil errors. var errStrings []string for _, err := range errs { if err != nil { errStrings = append(errStrings, err.Error()) } } // Return nil if there are no non-nil errors in the input. if len(errStrings) <= 0 { return nil } // Combine all of the non-nil errors into one larger return value. return errors.New(strings.Join(errStrings, "; ")) } // extendErr will return an error that is the same type as the input error, but // prefixed with the provided context. This only works for the error types // defined in the host package. If the input error is nil, the extension is // ignored and nil will be returned. func extendErr(s string, err error) error { if err == nil { return nil } switch v := err.(type) { case ErrorCommunication: return ErrorCommunication(s) + v case ErrorConnection: return ErrorConnection(s) + v case ErrorConsensus: return ErrorConsensus(s) + v case ErrorInternal: return ErrorInternal(s) + v default: return errors.New(s + err.Error()) } } // Error satisfies the Error interface for the ErrorCommunication type. func (ec ErrorCommunication) Error() string { return "communication error: " + string(ec) } // Error satisfies the Error interface for the ErrorConnection type. func (ec ErrorConnection) Error() string { return "connection error: " + string(ec) } // Error satisfies the Error interface for the ErrorConsensus type. func (ec ErrorConsensus) Error() string { return "consensus error: " + string(ec) } // Error satisfies the Error interface for the ErrorInternal type. func (ec ErrorInternal) Error() string { return "internal error: " + string(ec) } // mangedLogError will take an error and log it to the host, depending on the // type of error and whether or not the DEBUG flag has been set. func (h *Host) managedLogError(err error) { // Determine the type of error and the number of times that this error has // been logged. var num uint64 var probability int // Error will be logged with 1/probability chance. switch err.(type) { case ErrorCommunication: num = atomic.LoadUint64(&h.atomicCommunicationErrors) probability = errorCommunicationProbability case ErrorConnection: num = atomic.LoadUint64(&h.atomicConnectionErrors) probability = errorConnectionProbability case ErrorConsensus: num = atomic.LoadUint64(&h.atomicConsensusErrors) probability = errorConsensusProbability case ErrorInternal: num = atomic.LoadUint64(&h.atomicInternalErrors) probability = errorInternalProbability default: num = atomic.LoadUint64(&h.atomicNormalErrors) probability = errorNormalProbability } // If num > logFewLimit, substantially decrease the probability that the error // gets logged. if num > logFewLimit { probability = probability * 25 } // If we've seen less than logAllLimit of that type of error before, log // the error as a normal logging statement. Otherwise, probabilistically // log the statement. In debugging mode, log all statements. logged := false rand, randErr := crypto.RandIntn(probability + 1) if randErr != nil { h.log.Critical("random number generation failed") } if num < logAllLimit || rand == probability { logged = true h.log.Println(err) } else { h.log.Debugln(err) } // If the error was logged, increment the log counter. if logged { switch err.(type) { case ErrorCommunication: atomic.AddUint64(&h.atomicCommunicationErrors, 1) case ErrorConnection: atomic.AddUint64(&h.atomicConnectionErrors, 1) case ErrorConsensus: atomic.AddUint64(&h.atomicConsensusErrors, 1) case ErrorInternal: atomic.AddUint64(&h.atomicInternalErrors, 1) default: atomic.AddUint64(&h.atomicNormalErrors, 1) } } } Sia-1.0.4/modules/host/errors_test.go000066400000000000000000000207441302033070500175570ustar00rootroot00000000000000package host import ( "bufio" "errors" "os" "path/filepath" "strings" "testing" "github.com/NebulousLabs/Sia/modules" ) // countFileLines is a helper function that will count the number of lines in a // file, based on the number of '\n' characters. countFileLines will load the // file into memory using ioutil.ReadAll. // // countFileLines will ignore all lines with the string 'DEBUG' in it. func countFileLines(filepath string) (uint64, error) { file, err := os.Open(filepath) if err != nil { return 0, err } scanner := bufio.NewScanner(file) lines := uint64(0) for scanner.Scan() { line := scanner.Text() if !strings.Contains(line, "[DEBUG]") { lines++ } } return lines, nil } // TestComposeErrors checks that composeErrors is correctly composing errors // and handling edge cases. func TestComposeErrors(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() trials := []struct { inputErrors []error nilReturn bool expectedComposedError string }{ { nil, true, "", }, { make([]error, 0), true, "", }, { []error{errors.New("single error")}, false, "single error", }, { []error{ errors.New("first error"), errors.New("second error"), }, false, "first error; second error", }, { []error{ errors.New("first error"), errors.New("second error"), errors.New("third error"), }, false, "first error; second error; third error", }, { []error{ nil, errors.New("second error"), errors.New("third error"), }, false, "second error; third error", }, { []error{ errors.New("first error"), nil, nil, }, false, "first error", }, { []error{ nil, nil, nil, }, true, "", }, } for _, trial := range trials { err := composeErrors(trial.inputErrors...) if trial.nilReturn { if err != nil { t.Error("composeError failed a test, expecting nil, got", err) } } else { if err == nil { t.Error("not expecting a nil error when doing composition") } if err.Error() != trial.expectedComposedError { t.Error("composeError failed a test, expecting", trial.expectedComposedError, "got", err.Error()) } } } } // TestExtendErr checks that extendErr works as described - preserving the // error type within the package and adding a string. Also returning nil if the // input error is nil. func TestExtendErr(t *testing.T) { // Try extending a nil error. var err error err2 := extendErr("extend: ", err) if err2 != nil { t.Error("providing a nil error to extendErr does not return nil") } // Try extending a normal error. err = errors.New("extend me") err2 = extendErr("extend: ", err) if err2.Error() != "extend: extend me" { t.Error("normal error not extended correctly") } // Try extending ErrorCommunication. err = ErrorCommunication("err") err2 = extendErr("extend: ", err) if err2.Error() != "communication error: extend: err" { t.Error("extending ErrorCommunication did not occur correctly:", err2.Error()) } if _, ok := err2.(ErrorCommunication); !ok { t.Error("extended error did not preserve error type") } // Try extending ErrorConnection. err = ErrorConnection("err") err2 = extendErr("extend: ", err) if err2.Error() != "connection error: extend: err" { t.Error("extending ErrorConnection did not occur correctly:", err2.Error()) } switch err2.(type) { case ErrorConnection: default: t.Error("extended error did not preserve error type") } // Try extending ErrorConsensus. err = ErrorConsensus("err") err2 = extendErr("extend: ", err) if err2.Error() != "consensus error: extend: err" { t.Error("extending ErrorConsensus did not occur correctly:", err2.Error()) } switch err2.(type) { case ErrorConsensus: default: t.Error("extended error did not preserve error type") } // Try extending ErrorInternal. err = ErrorInternal("err") err2 = extendErr("extend: ", err) if err2.Error() != "internal error: extend: err" { t.Error("extending ErrorInternal did not occur correctly:", err2.Error()) } switch err2.(type) { case ErrorInternal: default: t.Error("extended error did not preserve error type") } } // TestManagedLogError will check that errors are being logged correctly based // on the logAllLimit, the probabilities, and the logFewLimit. func TestManagedLogError(t *testing.T) { if testing.Short() { t.SkipNow() } ht, err := newHostTester("TestManagedLogError") if err != nil { t.Fatal(err) } defer ht.Close() logFilepath := filepath.Join(ht.persistDir, modules.HostDir, logFile) // Count the number of lines in the log file. baseLines, err := countFileLines(logFilepath) if err != nil { t.Fatal(err) } // Log 'logAllLimit' for ErrorCommunication. for i := uint64(0); i < logAllLimit; i++ { ht.host.managedLogError(ErrorCommunication("comm error")) } logLines, err := countFileLines(logFilepath) if err != nil { t.Fatal(err) } if logLines != baseLines+logAllLimit { t.Error("does not seem that all communication errors were logged") } baseLines = logLines // Log 'logAllLimit' for ErrorConnection. for i := uint64(0); i < logAllLimit; i++ { ht.host.managedLogError(ErrorConnection("conn error")) } logLines, err = countFileLines(logFilepath) if err != nil { t.Fatal(err) } if logLines != baseLines+logAllLimit { t.Error("does not seem that all connection errors were logged") } baseLines = logLines // Log 'logAllLimit' for ErrorConsensus. for i := uint64(0); i < logAllLimit; i++ { ht.host.managedLogError(ErrorConsensus("consensus error")) } logLines, err = countFileLines(logFilepath) if err != nil { t.Fatal(err) } if logLines != baseLines+logAllLimit { t.Error("does not seem that all consensus errors were logged") } baseLines = logLines // Log 'logAllLimit' for ErrorInternal. for i := uint64(0); i < logAllLimit; i++ { ht.host.managedLogError(ErrorInternal("internal error")) } logLines, err = countFileLines(logFilepath) if err != nil { t.Fatal(err) } if logLines != baseLines+logAllLimit { t.Error("does not seem that all internal errors were logged") } baseLines = logLines // Log 'logAllLimit' for normal errors. for i := uint64(0); i < logAllLimit; i++ { ht.host.managedLogError(errors.New("normal error")) } logLines, err = countFileLines(logFilepath) if err != nil { t.Fatal(err) } if logLines != baseLines+logAllLimit { t.Error("does not seem that all normal errors were logged", logLines, baseLines, logAllLimit) } baseLines = logLines // Log enough ErrorInternal errors to bring ErrorInternal close, but not // all the way, to the 'logFewLimit'. remaining := logFewLimit - logAllLimit logsNeeded := remaining * errorInternalProbability for i := uint64(0); i < logsNeeded/3; i++ { ht.host.managedLogError(ErrorInternal("internal err")) } logLines, err = countFileLines(logFilepath) if err != nil { t.Fatal(err) } if logLines < baseLines+remaining/6 || logLines > baseLines+remaining { t.Error("probabilistic logging is not logging with the correct probability:", logLines, baseLines, remaining) } // Log enough ErrorInternal errors to bring it all the way to // 'logFewLimit'. for i := uint64(0); i < logsNeeded*5; i++ { ht.host.managedLogError(ErrorInternal("internal err")) } logLines, err = countFileLines(logFilepath) if err != nil { t.Fatal(err) } if logLines < baseLines+remaining || logLines > baseLines+logsNeeded*2 { t.Error("probabilisitic logging is not clamping correctly:", baseLines, logLines, logsNeeded) } baseLines = logLines // Log enough ErrorCommunication errors to bring ErrorCommunication close, but not // all the way, to the 'logFewLimit'. remaining = logFewLimit - logAllLimit logsNeeded = remaining * errorCommunicationProbability for i := uint64(0); i < logsNeeded/3; i++ { ht.host.managedLogError(ErrorCommunication("comm err")) } logLines, err = countFileLines(logFilepath) if err != nil { t.Fatal(err) } if logLines < baseLines+remaining/6 || logLines > baseLines+remaining { t.Error("probabilistic logging is not logging with the correct probability:", baseLines, logLines, logsNeeded) } // Log enough ErrorCommunication errors to bring it all the way to // 'logFewLimit'. for i := uint64(0); i < logsNeeded*5; i++ { ht.host.managedLogError(ErrorCommunication("comm err")) } logLines, err = countFileLines(logFilepath) if err != nil { t.Fatal(err) } if logLines < baseLines+remaining || logLines > baseLines+logsNeeded*2 { t.Error("probabilisitic logging is not clamping correctly:", baseLines, logLines, logsNeeded) } } Sia-1.0.4/modules/host/host.go000066400000000000000000000317161302033070500161620ustar00rootroot00000000000000// Package host is an implementation of the host module, and is responsible for // participating in the storage ecosystem, turning available disk space an // internet bandwidth into profit for the user. package host // TODO: what happens if the renter submits the revision early, before the // final revision. Will the host mark the contract as complete? // TODO: Host and renter are reporting errors where the renter is not adding // enough fees to the file contract. // TODO: Test the safety of the builder, it should be okay to have multiple // builders open for up to 600 seconds, which means multiple blocks could be // received in that time period. Should also check what happens if a prent gets // confirmed on the blockchain before the builder is finished. // TODO: Double check that any network connection has a finite deadline - // handling action items properly requires that the locks held on the // obligations eventually be released. There's also some more advanced // implementation that needs to happen with the storage obligation locks to // make sure that someone who wants a lock is able to get it eventually. // TODO: Add contract compensation from form contract to the storage obligation // financial metrics, and to the host's tracking. // TODO: merge the network interfaces stuff, don't forget to include the // 'announced' variable as one of the outputs. // TODO: 'announced' doesn't tell you if the announcement made it to the // blockchain. // TODO: Need to make sure that the revision exchange for the renter and the // host is being handled correctly. For the host, it's not so difficult. The // host need only send the most recent revision every time. But, the host // should not sign a revision unless the renter has explicitly signed such that // the 'WholeTransaction' fields cover only the revision and that the // signatures for the revision don't depend on anything else. The renter needs // to verify the same when checking on a file contract revision from the host. // If the host has submitted a file contract revision where the signatures have // signed the whole file contract, there is an issue. // TODO: there is a mistake in the file contract revision rpc, the host, if it // does not have the right file contract id, should be returning an error there // to the renter (and not just to it's calling function without informing the // renter what's up). // TODO: Need to make sure that the correct height is being used when adding // sectors to the storage manager - in some places right now WindowStart is // being used but really it's WindowEnd that should be in use. // TODO: The host needs some way to blacklist file contracts that are being // abusive by repeatedly getting free download batches. // TODO: clean up all of the magic numbers in the host. // TODO: revamp the finances for the storage obligations. // TODO: host_test.go has commented out tests. // TODO: network_test.go has commented out tests. // TODO: persist_test.go has commented out tests. // TODO: update_test.go has commented out tests. import ( "errors" "fmt" "net" "path/filepath" "sync" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/host/storagemanager" "github.com/NebulousLabs/Sia/persist" siasync "github.com/NebulousLabs/Sia/sync" "github.com/NebulousLabs/Sia/types" ) const ( // Names of the various persistent files in the host. dbFilename = modules.HostDir + ".db" logFile = modules.HostDir + ".log" settingsFile = modules.HostDir + ".json" ) var ( // dbMetadata is a header that gets put into the database to identify a // version and indicate that the database holds host information. dbMetadata = persist.Metadata{ Header: "Sia Host DB", Version: "0.5.2", } // persistMetadata is the header that gets written to the persist file, and is // used to recognize other persist files. persistMetadata = persist.Metadata{ Header: "Sia Host", Version: "0.5", } // errHostClosed gets returned when a call is rejected due to the host // having been closed. errHostClosed = errors.New("call is disabled because the host is closed") // Nil dependency errors. errNilCS = errors.New("host cannot use a nil state") errNilTpool = errors.New("host cannot use a nil transaction pool") errNilWallet = errors.New("host cannot use a nil wallet") ) // A Host contains all the fields necessary for storing files for clients and // performing the storage proofs on the received files. type Host struct { // RPC Metrics - atomic variables need to be placed at the top to preserve // compatibility with 32bit systems. atomicDownloadCalls uint64 atomicErroredCalls uint64 atomicFormContractCalls uint64 atomicRenewCalls uint64 atomicReviseCalls uint64 atomicRecentRevisionCalls uint64 atomicSettingsCalls uint64 atomicUnrecognizedCalls uint64 // Error management. There are a few different types of errors returned by // the host. These errors intentionally not persistent, so that the logging // limits of each error type will be reset each time the host is reset. atomicCommunicationErrors uint64 atomicConnectionErrors uint64 atomicConsensusErrors uint64 atomicInternalErrors uint64 atomicNormalErrors uint64 // Dependencies. cs modules.ConsensusSet tpool modules.TransactionPool wallet modules.Wallet dependencies modules.StorageManager // Consensus Tracking. blockHeight types.BlockHeight recentChange modules.ConsensusChangeID // Host Identity // // The revision number keeps track of the current revision number on the // host external settingse // // The auto address is the address that is fetched automatically by the // host. The host will ignore the automatic address if settings.NetAddress // has been set by the user. If settings.NetAddress is blank, then the host // will track its own ip address and make an announcement on the blockchain // every time that the address changes. // // The announced bool indicates whether the host remembers having a // successful announcement with the current address. announced bool autoAddress modules.NetAddress financialMetrics modules.HostFinancialMetrics publicKey types.SiaPublicKey revisionNumber uint64 secretKey crypto.SecretKey settings modules.HostInternalSettings unlockHash types.UnlockHash // A wallet address that can receive coins. // A map of storage obligations that are currently being modified. Locks on // storage obligations can be long-running, and each storage obligation can // be locked separately. lockedStorageObligations map[types.FileContractID]*siasync.TryMutex // Utilities. db *persist.BoltDatabase listener net.Listener log *persist.Logger mu sync.RWMutex persistDir string port string tg siasync.ThreadGroup } // checkUnlockHash will check that the host has an unlock hash. If the host // does not have an unlock hash, an attempt will be made to get an unlock hash // from the wallet. That may fail due to the wallet being locked, in which case // an error is returned. func (h *Host) checkUnlockHash() error { if h.unlockHash == (types.UnlockHash{}) { uc, err := h.wallet.NextAddress() if err != nil { return err } // Set the unlock hash and save the host. Saving is important, because // the host will be using this unlock hash to establish identity, and // losing it will mean silently losing part of the host identity. h.unlockHash = uc.UnlockHash() err = h.saveSync() if err != nil { return err } } return nil } // newHost returns an initialized Host, taking a set of dependencies as input. // By making the dependencies an argument of the 'new' call, the host can be // mocked such that the dependencies can return unexpected errors or unique // behaviors during testing, enabling easier testing of the failure modes of // the Host. func newHost(dependencies dependencies, cs modules.ConsensusSet, tpool modules.TransactionPool, wallet modules.Wallet, listenerAddress string, persistDir string) (*Host, error) { // Check that all the dependencies were provided. if cs == nil { return nil, errNilCS } if tpool == nil { return nil, errNilTpool } if wallet == nil { return nil, errNilWallet } // Create the host object. h := &Host{ cs: cs, tpool: tpool, wallet: wallet, dependencies: dependencies, lockedStorageObligations: make(map[types.FileContractID]*siasync.TryMutex), persistDir: persistDir, } // Call stop in the event of a partial startup. var err error defer func() { if err != nil { err = composeErrors(h.tg.Stop(), err) } }() // Create the perist directory if it does not yet exist. err = dependencies.mkdirAll(h.persistDir, 0700) if err != nil { return nil, err } // Initialize the logger, and set up the stop call that will close the // logger. h.log, err = dependencies.newLogger(filepath.Join(h.persistDir, logFile)) if err != nil { return nil, err } h.tg.AfterStop(func() { err = h.log.Close() if err != nil { // State of the logger is uncertain, a Println will have to // suffice. fmt.Println("Error when closing the logger:", err) } }) // Add the storage manager to the host, and set up the stop call that will // close the storage manager. h.StorageManager, err = storagemanager.New(filepath.Join(persistDir, "storagemanager")) if err != nil { h.log.Println("Could not open the storage manager:", err) return nil, err } h.tg.AfterStop(func() { err = h.StorageManager.Close() if err != nil { h.log.Println("Could not close storage manager:", err) } }) // After opening the database, it must be initialized. Most commonly, // nothing happens. But for new databases, a set of buckets must be // created. Initialization is also a good time to run sanity checks. err = h.initDB() if err != nil { h.log.Println("Could not initalize database:", err) return nil, err } // Load the prior persistence structures, and configure the host to save // before shutting down. err = h.load() if err != nil { return nil, err } h.tg.AfterStop(func() { err = h.saveSync() if err != nil { h.log.Println("Could not save host upon shutdown:", err) } }) // Initialize the networking. err = h.initNetworking(listenerAddress) if err != nil { h.log.Println("Could not initialize host networking:", err) return nil, err } return h, nil } // New returns an initialized Host. func New(cs modules.ConsensusSet, tpool modules.TransactionPool, wallet modules.Wallet, address string, persistDir string) (*Host, error) { return newHost(productionDependencies{}, cs, tpool, wallet, address, persistDir) } // Close shuts down the host. func (h *Host) Close() error { return h.tg.Stop() } // ExternalSettings returns the hosts external settings. These values cannot be // set by the user (host is configured through InternalSettings), and are the // values that get displayed to other hosts on the network. func (h *Host) ExternalSettings() modules.HostExternalSettings { h.mu.RLock() defer h.mu.RUnlock() err := h.tg.Add() if err != nil { build.Critical("Call to ExternalSettings after close") } defer h.tg.Done() return h.externalSettings() } // FinancialMetrics returns information about the financial commitments, // rewards, and activities of the host. func (h *Host) FinancialMetrics() modules.HostFinancialMetrics { h.mu.RLock() defer h.mu.RUnlock() err := h.tg.Add() if err != nil { build.Critical("Call to FinancialMetrics after close") } defer h.tg.Done() return h.financialMetrics } // SetInternalSettings updates the host's internal HostInternalSettings object. func (h *Host) SetInternalSettings(settings modules.HostInternalSettings) error { h.mu.Lock() defer h.mu.Unlock() err := h.tg.Add() if err != nil { return err } defer h.tg.Done() // The host should not be accepting file contracts if it does not have an // unlock hash. if settings.AcceptingContracts { err := h.checkUnlockHash() if err != nil { return errors.New("internal settings not updated, no unlock hash: " + err.Error()) } } if settings.NetAddress != "" { err := settings.NetAddress.IsValid() if err != nil { return errors.New("internal settings not updated, invalid NetAddress: " + err.Error()) } } // Check if the net address for the host has changed. If it has, and it's // not equal to the auto address, then the host is going to need to make // another blockchain announcement. if h.settings.NetAddress != settings.NetAddress && settings.NetAddress != h.autoAddress { h.announced = false } h.settings = settings h.revisionNumber++ err = h.saveSync() if err != nil { return errors.New("internal settings updated, but failed saving to disk: " + err.Error()) } return nil } // InternalSettings returns the settings of a host. func (h *Host) InternalSettings() modules.HostInternalSettings { h.mu.RLock() defer h.mu.RUnlock() err := h.tg.Add() if err != nil { return modules.HostInternalSettings{} } defer h.tg.Done() return h.settings } Sia-1.0.4/modules/host/host_errors_test.go000066400000000000000000000136171302033070500206150ustar00rootroot00000000000000package host import ( "net" "os" "path/filepath" "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" ) // dependencyErrMkdirAll is a dependency set that returns an error when MkdirAll // is called. type dependencyErrMkdirAll struct { productionDependencies } func (dependencyErrMkdirAll) mkdirAll(string, os.FileMode) error { return mockErrMkdirAll } // TestHostFailedMkdirAll initializes the host using a call to MkdirAll that // will fail. func TestHostFailedMkdirAll(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := blankHostTester("TestHostFailedMkdirAll") if err != nil { t.Fatal(err) } defer ht.Close() err = ht.host.Close() if err != nil { t.Fatal(err) } ht.host, err = newHost(dependencyErrMkdirAll{}, ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != mockErrMkdirAll { t.Fatal(err) } // Set ht.host to something non-nil - nil was returned because startup was // incomplete. If ht.host is nil at the end of the function, the ht.Close() // operation will fail. ht.host, err = newHost(productionDependencies{}, ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != nil { t.Fatal(err) } } // dependencyErrNewLogger is a dependency set that returns an error when // NewLogger is called. type dependencyErrNewLogger struct { productionDependencies } func (dependencyErrNewLogger) newLogger(string) (*persist.Logger, error) { return nil, mockErrNewLogger } // TestHostFailedNewLogger initializes the host using a call to NewLogger that // will fail. func TestHostFailedNewLogger(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := blankHostTester("TestHostFailedNewLogger") if err != nil { t.Fatal(err) } defer ht.Close() err = ht.host.Close() if err != nil { t.Fatal(err) } ht.host, err = newHost(dependencyErrNewLogger{}, ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != mockErrNewLogger { t.Fatal(err) } // Set ht.host to something non-nil - nil was returned because startup was // incomplete. If ht.host is nil at the end of the function, the ht.Close() // operation will fail. ht.host, err = newHost(productionDependencies{}, ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != nil { t.Fatal(err) } } // dependencyErrOpenDatabase is a dependency that returns an error when // OpenDatabase is called. type dependencyErrOpenDatabase struct { productionDependencies } func (dependencyErrOpenDatabase) openDatabase(persist.Metadata, string) (*persist.BoltDatabase, error) { return nil, mockErrOpenDatabase } // TestHostFailedOpenDatabase initializes the host using a call to OpenDatabase // that has been mocked to fail. func TestHostFailedOpenDatabase(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := blankHostTester("TestHostFailedOpenDatabase") if err != nil { t.Fatal(err) } defer ht.Close() err = ht.host.Close() if err != nil { t.Fatal(err) } ht.host, err = newHost(dependencyErrOpenDatabase{}, ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != mockErrOpenDatabase { t.Fatal(err) } // Set ht.host to something non-nil - nil was returned because startup was // incomplete. If ht.host is nil at the end of the function, the ht.Close() // operation will fail. ht.host, err = newHost(productionDependencies{}, ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != nil { t.Fatal(err) } } // dependencyErrLoadFile is a dependency that returns an error when // LoadFile is called. type dependencyErrLoadFile struct { productionDependencies } func (dependencyErrLoadFile) loadFile(persist.Metadata, interface{}, string) error { return mockErrLoadFile } // TestHostFailedLoadFile initializes the host using a call to LoadFile that // has been mocked to fail. func TestHostFailedLoadFile(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := blankHostTester("TestHostFailedLoadFile") if err != nil { t.Fatal(err) } defer ht.Close() err = ht.host.Close() if err != nil { t.Fatal(err) } ht.host, err = newHost(dependencyErrLoadFile{}, ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != mockErrLoadFile { t.Fatal(err) } // Set ht.host to something non-nil - nil was returned because startup was // incomplete. If ht.host is nil at the end of the function, the ht.Close() // operation will fail. ht.host, err = newHost(productionDependencies{}, ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != nil { t.Fatal(err) } } // dependencyErrListen is a dependency that returns an error when Listen is // called. type dependencyErrListen struct { productionDependencies } func (dependencyErrListen) listen(string, string) (net.Listener, error) { return nil, mockErrListen } // TestHostFailedListen initializes the host using a call to Listen that // has been mocked to fail. func TestHostFailedListen(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := blankHostTester("TestHostFailedListen") if err != nil { t.Fatal(err) } defer ht.Close() err = ht.host.Close() if err != nil { t.Fatal(err) } ht.host, err = newHost(dependencyErrListen{}, ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != mockErrListen { t.Fatal(err) } // Set ht.host to something non-nil - nil was returned because startup was // incomplete. If ht.host is nil at the end of the function, the ht.Close() // operation will fail. ht.host, err = newHost(productionDependencies{}, ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != nil { t.Fatal(err) } } Sia-1.0.4/modules/host/host_test.go000066400000000000000000000401031302033070500172070ustar00rootroot00000000000000package host import ( // "errors" "os" "path/filepath" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/consensus" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/modules/miner" // "github.com/NebulousLabs/Sia/modules/renter" "github.com/NebulousLabs/Sia/modules/transactionpool" "github.com/NebulousLabs/Sia/modules/wallet" siasync "github.com/NebulousLabs/Sia/sync" "github.com/NebulousLabs/Sia/types" ) // A hostTester is the helper object for host testing, including helper modules // and methods for controlling synchronization. type hostTester struct { cs modules.ConsensusSet gateway modules.Gateway miner modules.TestMiner // renter modules.Renter renting bool tpool modules.TransactionPool wallet modules.Wallet walletKey crypto.TwofishKey host *Host persistDir string } /* // initRenting prepares the host tester for uploads and downloads by announcing // the host to the network and performing other preparational tasks. // initRenting takes a while because the renter needs to process the host // announcement, requiring asynchronous network communication between the // renter and host. func (ht *hostTester) initRenting() error { if ht.renting { return nil } // Because the renting test takes a long time, it will fail if // testing.Short. if testing.Short() { return errors.New("cannot call initRenting in short tests") } // Announce the host. err := ht.host.Announce() if err != nil { return err } // Mine a block to get the announcement into the blockchain. _, err = ht.miner.AddBlock() if err != nil { return err } // Wait for the renter to see the host announcement. for i := 0; i < 50; i++ { time.Sleep(time.Millisecond * 100) if len(ht.renter.ActiveHosts()) != 0 { break } } if len(ht.renter.ActiveHosts()) == 0 { return errors.New("could not start renting in the host tester") } ht.renting = true return nil } */ // initWallet creates a wallet key, initializes the host wallet, unlocks it, // and then stores the key in the host tester. func (ht *hostTester) initWallet() error { // Create the keys for the wallet and unlock it. key, err := crypto.GenerateTwofishKey() if err != nil { return err } ht.walletKey = key _, err = ht.wallet.Encrypt(key) if err != nil { return err } err = ht.wallet.Unlock(key) if err != nil { return err } return nil } // blankHostTester creates a host tester where the modules are created but no // extra initialization has been done, for example no blocks have been mined // and the wallet keys have not been created. func blankHostTester(name string) (*hostTester, error) { return blankMockHostTester(productionDependencies{}, name) } // blankMockHostTester creates a host tester where the modules are created but no // extra initialization has been done, for example no blocks have been mined // and the wallet keys have not been created. func blankMockHostTester(d dependencies, name string) (*hostTester, error) { testdir := build.TempDir(modules.HostDir, name) // Create the modules. g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, err } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { return nil, err } w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.MinerDir)) if err != nil { return nil, err } h, err := newHost(d, cs, tp, w, "localhost:0", filepath.Join(testdir, modules.HostDir)) if err != nil { return nil, err } /* r, err := renter.New(cs, w, tp, filepath.Join(testdir, modules.RenterDir)) if err != nil { return nil, err } */ // Assemble all objects into a hostTester ht := &hostTester{ cs: cs, gateway: g, miner: m, // renter: r, tpool: tp, wallet: w, host: h, persistDir: testdir, } return ht, nil } // newHostTester creates a host tester with an initialized wallet and money in // that wallet. func newHostTester(name string) (*hostTester, error) { return newMockHostTester(productionDependencies{}, name) } // newMockHostTester creates a host tester with an initialized wallet and money // in that wallet, using the dependencies provided. func newMockHostTester(d dependencies, name string) (*hostTester, error) { // Create a blank host tester. ht, err := blankMockHostTester(d, name) if err != nil { return nil, err } // Initialize the wallet and mine blocks until the wallet has money. err = ht.initWallet() if err != nil { return nil, err } for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ { _, err = ht.miner.AddBlock() if err != nil { return nil, err } } // Create two storage folder for the host, one the minimum size and one // twice the minimum size. storageFolderOne := filepath.Join(ht.persistDir, "hostTesterStorageFolderOne") err = os.Mkdir(storageFolderOne, 0700) if err != nil { return nil, err } err = ht.host.AddStorageFolder(storageFolderOne, modules.SectorSize*8) if err != nil { return nil, err } storageFolderTwo := filepath.Join(ht.persistDir, "hostTesterStorageFolderTwo") err = os.Mkdir(storageFolderTwo, 0700) if err != nil { return nil, err } err = ht.host.AddStorageFolder(storageFolderTwo, modules.SectorSize*8*2) if err != nil { return nil, err } return ht, nil } // Close safely closes the hostTester. It panics if err != nil because there // isn't a good way to errcheck when deferring a close. func (ht *hostTester) Close() error { errs := []error{ ht.host.Close(), ht.miner.Close(), ht.tpool.Close(), ht.cs.Close(), ht.gateway.Close(), } if err := build.JoinErrors(errs, "; "); err != nil { panic(err) } return nil } // TestHostInitialization checks that the host initializes to sensible default // values. func TestHostInitialization(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // Create a blank host tester and check that the height is zero. bht, err := blankHostTester("TestHostInitialization") if err != nil { t.Fatal(err) } defer bht.Close() if bht.host.blockHeight != 0 { t.Error("host initialized to the wrong block height") } // Initialize the wallet so that a block can be mined, then mine a block // and check that it sets the host height to 1. err = bht.initWallet() if err != nil { t.Fatal(err) } _, err = bht.miner.AddBlock() if err != nil { t.Fatal(err) } if bht.host.blockHeight != 1 { t.Fatal("block height did not increase correctly after first block mined:", bht.host.blockHeight, 1) } } // TestHostMultiClose checks that the host returns an error if Close is called // multiple times on the host. func TestHostMultiClose(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := newHostTester("TestHostMultiClose") if err != nil { t.Fatal(err) } defer ht.Close() err = ht.host.Close() if err != nil { t.Fatal(err) } err = ht.host.Close() if err != siasync.ErrStopped { t.Fatal(err) } err = ht.host.Close() if err != siasync.ErrStopped { t.Fatal(err) } // Set ht.host to something non-nil - nil was returned because startup was // incomplete. If ht.host is nil at the end of the function, the ht.Close() // operation will fail. ht.host, err = newHost(productionDependencies{}, ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != nil { t.Fatal(err) } } // TestNilValues tries initializing the host with nil values. func TestNilValues(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := blankHostTester("TestStartupRescan") if err != nil { t.Fatal(err) } defer ht.Close() hostDir := filepath.Join(ht.persistDir, modules.HostDir) _, err = New(nil, ht.tpool, ht.wallet, "localhost:0", hostDir) if err != errNilCS { t.Fatal("could not trigger errNilCS") } _, err = New(ht.cs, nil, ht.wallet, "localhost:0", hostDir) if err != errNilTpool { t.Fatal("could not trigger errNilTpool") } _, err = New(ht.cs, ht.tpool, nil, "localhost:0", hostDir) if err != errNilWallet { t.Fatal("Could not trigger errNilWallet") } } // TestSetAndGetInternalSettings checks that the functions for interacting with // the host's internal settings object are working as expected. func TestSetAndGetInternalSettings(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := newHostTester("TestSetAndGetInternalSettings") if err != nil { t.Fatal(err) } defer ht.Close() // Check the default settings get returned at first call. settings := ht.host.InternalSettings() if settings.AcceptingContracts != false { t.Error("settings retrieval did not return default value") } if settings.MaxDuration != defaultMaxDuration { t.Error("settings retrieval did not return default value") } if settings.MaxDownloadBatchSize != uint64(defaultMaxDownloadBatchSize) { t.Error("settings retrieval did not return default value") } if settings.MaxReviseBatchSize != uint64(defaultMaxReviseBatchSize) { t.Error("settings retrieval did not return default value") } if settings.NetAddress != "" { t.Error("settings retrieval did not return default value") } if settings.WindowSize != defaultWindowSize { t.Error("settings retrieval did not return default value") } if !settings.Collateral.Equals(defaultCollateral) { t.Error("settings retrieval did not return default value") } if !settings.CollateralBudget.Equals(defaultCollateralBudget) { t.Error("settings retrieval did not return default value") } if !settings.MaxCollateral.Equals(defaultMaxCollateral) { t.Error("settings retrieval did not return default value") } if !settings.MinContractPrice.Equals(defaultContractPrice) { t.Error("settings retrieval did not return default value") } if !settings.MinDownloadBandwidthPrice.Equals(defaultDownloadBandwidthPrice) { t.Error("settings retrieval did not return default value") } if !settings.MinStoragePrice.Equals(defaultStoragePrice) { t.Error("settings retrieval did not return default value") } if !settings.MinUploadBandwidthPrice.Equals(defaultUploadBandwidthPrice) { t.Error("settings retrieval did not return default value") } // Check that calling SetInternalSettings with valid settings updates the settings. settings.AcceptingContracts = true settings.NetAddress = "foo.com:123" err = ht.host.SetInternalSettings(settings) if err != nil { t.Fatal(err) } settings = ht.host.InternalSettings() if settings.AcceptingContracts != true { t.Fatal("SetInternalSettings failed to update settings") } if settings.NetAddress != "foo.com:123" { t.Fatal("SetInternalSettings failed to update settings") } // Check that calling SetInternalSettings with invalid settings does not update the settings. settings.NetAddress = "invalid" err = ht.host.SetInternalSettings(settings) if err == nil { t.Fatal("expected SetInternalSettings to error with invalid settings") } settings = ht.host.InternalSettings() if settings.NetAddress != "foo.com:123" { t.Fatal("SetInternalSettings should not modify the settings if the new settings are invalid") } // Reload the host and verify that the altered settings persisted. err = ht.host.Close() if err != nil { t.Fatal(err) } rebootHost, err := New(ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != nil { t.Fatal(err) } rebootSettings := rebootHost.InternalSettings() if rebootSettings.AcceptingContracts != settings.AcceptingContracts { t.Error("settings retrieval did not return updated value") } if rebootSettings.NetAddress != settings.NetAddress { t.Error("settings retrieval did not return updated value") } // Set ht.host to 'rebootHost' so that the 'ht.Close()' method will close // everything cleanly. ht.host = rebootHost } /* // TestSetAndGetSettings checks that the functions for interacting with the // hosts settings object are working as expected. func TestSetAndGetSettings(t *testing.T) { if testing.Short() { t.SkipNow() } ht, err := newHostTester("TestSetAndGetSettings") if err != nil { t.Fatal(err) } defer ht.Close() // Check the default settings get returned at first call. settings := ht.host.Settings() if settings.MaxDuration != defaultMaxDuration { t.Error("settings retrieval did not return default value") } if settings.WindowSize != defaultWindowSize { t.Error("settings retrieval did not return default value") } if settings.Price.Cmp(defaultPrice) != 0 { t.Error("settings retrieval did not return default value") } if settings.Collateral.Cmp(defaultCollateral) != 0 { t.Error("settings retrieval did not return default value") } // Submit updated settings and check that the changes stuck. settings.TotalStorage += 15 settings.MaxDuration += 16 settings.WindowSize += 17 settings.Price = settings.Price.Add(types.NewCurrency64(18)) settings.Collateral = settings.Collateral.Add(types.NewCurrency64(19)) err = ht.host.SetSettings(settings) if err != nil { t.Fatal(err) } newSettings := ht.host.Settings() if settings.MaxDuration != newSettings.MaxDuration { t.Error("settings retrieval did not return updated value") } if settings.WindowSize != newSettings.WindowSize { t.Error("settings retrieval did not return updated value") } if settings.Price.Cmp(newSettings.Price) != 0 { t.Error("settings retrieval did not return updated value") } if settings.Collateral.Cmp(newSettings.Collateral) != 0 { t.Error("settings retrieval did not return updated value") } // Reload the host and verify that the altered settings persisted. err = ht.host.Close() if err != nil { t.Fatal(err) } rebootHost, err := New(ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != nil { t.Fatal(err) } rebootSettings := rebootHost.Settings() if settings.TotalStorage != rebootSettings.TotalStorage { t.Error("settings retrieval did not return updated value") } if settings.MaxDuration != rebootSettings.MaxDuration { t.Error("settings retrieval did not return updated value") } if settings.WindowSize != rebootSettings.WindowSize { t.Error("settings retrieval did not return updated value") } if settings.Price.Cmp(rebootSettings.Price) != 0 { t.Error("settings retrieval did not return updated value") } if settings.Collateral.Cmp(rebootSettings.Collateral) != 0 { t.Error("settings retrieval did not return updated value") } } // TestPersistentSettings checks that settings persist between instances of the // host. func TestPersistentSettings(t *testing.T) { if testing.Short() { t.SkipNow() } ht, err := newHostTester("TestSetPersistentSettings") if err != nil { t.Fatal(err) } defer ht.Close() // Submit updated settings. settings := ht.host.Settings() settings.TotalStorage += 25 settings.MaxDuration += 36 settings.WindowSize += 47 settings.Price = settings.Price.Add(types.NewCurrency64(38)) settings.Collateral = settings.Collateral.Add(types.NewCurrency64(99)) err = ht.host.SetSettings(settings) if err != nil { t.Fatal(err) } // Reboot the host and verify that the new settings stuck. err = ht.host.Close() // host saves upon closing if err != nil { t.Fatal(err) } h, err := New(ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != nil { t.Fatal(err) } newSettings := h.Settings() if settings.TotalStorage != newSettings.TotalStorage { t.Error("settings retrieval did not return updated value:", settings.TotalStorage, "vs", newSettings.TotalStorage) } if settings.MaxDuration != newSettings.MaxDuration { t.Error("settings retrieval did not return updated value") } if settings.WindowSize != newSettings.WindowSize { t.Error("settings retrieval did not return updated value") } if settings.Price.Cmp(newSettings.Price) != 0 { t.Error("settings retrieval did not return updated value") } if settings.Collateral.Cmp(newSettings.Collateral) != 0 { t.Error("settings retrieval did not return updated value") } } */ Sia-1.0.4/modules/host/negotiate.go000066400000000000000000000302371302033070500171610ustar00rootroot00000000000000package host import ( "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( // errBadContractOutputCounts is returned if the presented file contract // revision has the wrong number of outputs for either the valid or the // missed proof outputs. errBadContractOutputCounts = ErrorCommunication("rejected for having an unexpected number of outputs") // errBadContractParent is returned when a file contract revision is // presented which has a parent id that doesn't match the file contract // which is supposed to be getting revised. errBadContractParent = ErrorCommunication("could not find contract's parent") // errBadFileMerkleRoot is returned if the renter incorrectly updates the // file merkle root during a file contract revision. errBadFileMerkleRoot = ErrorCommunication("rejected for bad file merkle root") // errBadFileSize is returned if the renter incorrectly download and // changes the file size during a file contract revision. errBadFileSize = ErrorCommunication("rejected for bad file size") // errBadModificationIndex is returned if the renter requests a change on a // sector root that is not in the file contract. errBadModificationIndex = ErrorCommunication("renter has made a modification that points to a nonexistent sector") // errBadParentID is returned if the renter incorrectly download and // provides the wrong parent id during a file contract revision. errBadParentID = ErrorCommunication("rejected for bad parent id") // errBadPayoutUnlockHashes is returned if the renter incorrectly sets the // payout unlock hashes during contract formation. errBadPayoutUnlockHashes = ErrorCommunication("rejected for bad unlock hashes in the payout") // errBadRevisionNumber number is returned if the renter incorrectly // download and does not increase the revision number during a file // contract revision. errBadRevisionNumber = ErrorCommunication("rejected for bad revision number") // errBadSectorSize is returned if the renter provides a sector to be // inserted that is the wrong size. errBadSectorSize = ErrorCommunication("renter has provided an incorrectly sized sector") // errBadUnlockConditions is returned if the renter incorrectly download // and does not provide the right unlock conditions in the payment // revision. errBadUnlockConditions = ErrorCommunication("rejected for bad unlock conditions") // errBadUnlockHash is returned if the renter incorrectly updates the // unlock hash during a file contract revision. errBadUnlockHash = ErrorCommunication("rejected for bad new unlock hash") // errBadWindowEnd is returned if the renter incorrectly download and // changes the window end during a file contract revision. errBadWindowEnd = ErrorCommunication("rejected for bad new window end") // errBadWindowStart is returned if the renter incorrectly updates the // window start during a file contract revision. errBadWindowStart = ErrorCommunication("rejected for bad new window start") // errEarlyWindow is returned if the file contract provided by the renter // has a storage proof window that is starting too near in the future. errEarlyWindow = ErrorCommunication("rejected for a window that starts too soon") // errEmptyObject is returned if the renter sends an empty or nil object // unexpectedly. errEmptyObject = ErrorCommunication("renter has unexpectedly send an empty/nil object") // errHighRenterMissedOutput is returned if the renter incorrectly download // and deducts an insufficient amount from the renter missed outputs during // a file contract revision. errHighRenterMissedOutput = ErrorCommunication("rejected for high paying renter missed output") // errHighRenterValidOutput is returned if the renter incorrectly download // and deducts an insufficient amount from the renter valid outputs during // a file contract revision. errHighRenterValidOutput = ErrorCommunication("rejected for high paying renter valid output") // errIllegalOffsetAndLength is returned if the renter tries perform a // modify operation that uses a troublesome combination of offset and // length. errIllegalOffsetAndLength = ErrorCommunication("renter is trying to do a modify with an illegal offset and length") // errLargeSector is returned if the renter sends a RevisionAction that has // data which creates a sector that is larger than what the host uses. errLargeSector = ErrorCommunication("renter has sent a sector that exceeds the host's sector size") // errLateRevision is returned if the renter is attempting to revise a // revision after the revision deadline. The host needs time to submit the // final revision to the blockchain to guarantee payment, and therefore // will not accept revisions once the window start is too close. errLateRevision = ErrorCommunication("renter is requesting revision after the revision deadline") // errLongDuration is returned if the renter proposes a file contract with // an experation that is too far into the future according to the host's // settings. errLongDuration = ErrorCommunication("renter proposed a file contract with a too-long duration") // errLowTransactionFees is returned if the renter provides a transaction // that the host does not feel is able to make it onto the blockchain. errLowTransactionFees = ErrorCommunication("rejected for including too few transaction fees") // errLowHostMissedOutput is returned if the renter incorrectly updates the // host missed proof output during a file contract revision. errLowHostMissedOutput = ErrorCommunication("rejected for low paying host missed output") // errLowHostValidOutput is returned if the renter incorrectly updates the // host valid proof output during a file contract revision. errLowHostValidOutput = ErrorCommunication("rejected for low paying host valid output") // errLowVoidOutput is returned if the renter has not allocated enough // funds to the void output. errLowVoidOutput = ErrorCommunication("rejected for low value void output") // errMismatchedHostPayouts is returned if the renter incorrectly sets the // host valid and missed payouts to different values during contract // formation. errMismatchedHostPayouts = ErrorCommunication("rejected because host valid and missed payouts are not the same value") // errSmallWindow is returned if the renter suggests a storage proof window // that is too small. errSmallWindow = ErrorCommunication("rejected for small window size") // errUnknownModification is returned if the host receives a modification // action from the renter that it does not understand. errUnknownModification = ErrorCommunication("renter is attempting an action that the host does not understand") ) // createRevisionSignature creates a signature for a file contract revision // that signs on the file contract revision. The renter should have already // provided the signature. createRevisionSignature will check to make sure that // the renter's signature is valid. func createRevisionSignature(fcr types.FileContractRevision, renterSig types.TransactionSignature, secretKey crypto.SecretKey, blockHeight types.BlockHeight) (types.Transaction, error) { hostSig := types.TransactionSignature{ ParentID: crypto.Hash(fcr.ParentID), PublicKeyIndex: 1, CoveredFields: types.CoveredFields{ FileContractRevisions: []uint64{0}, }, } txn := types.Transaction{ FileContractRevisions: []types.FileContractRevision{fcr}, TransactionSignatures: []types.TransactionSignature{renterSig, hostSig}, } sigHash := txn.SigHash(1) encodedSig, err := crypto.SignHash(sigHash, secretKey) if err != nil { return types.Transaction{}, err } txn.TransactionSignatures[1].Signature = encodedSig[:] err = modules.VerifyFileContractRevisionTransactionSignatures(fcr, txn.TransactionSignatures, blockHeight) if err != nil { return types.Transaction{}, err } return txn, nil } // managedFinalizeContract will take a file contract, add the host's // collateral, and then try submitting the file contract to the transaction // pool. If there is no error, the completed transaction set will be returned // to the caller. func (h *Host) managedFinalizeContract(builder modules.TransactionBuilder, renterPK crypto.PublicKey, renterSignatures []types.TransactionSignature, renterRevisionSignature types.TransactionSignature, initialSectorRoots []crypto.Hash, hostCollateral, hostInitialRevenue, hostInitialRisk types.Currency) ([]types.TransactionSignature, types.TransactionSignature, types.FileContractID, error) { for _, sig := range renterSignatures { builder.AddTransactionSignature(sig) } fullTxnSet, err := builder.Sign(true) if err != nil { builder.Drop() return nil, types.TransactionSignature{}, types.FileContractID{}, err } // Verify that the signature for the revision from the renter is correct. h.mu.RLock() blockHeight := h.blockHeight hostSPK := h.publicKey hostSK := h.secretKey h.mu.RUnlock() contractTxn := fullTxnSet[len(fullTxnSet)-1] fc := contractTxn.FileContracts[0] noOpRevision := types.FileContractRevision{ ParentID: contractTxn.FileContractID(0), UnlockConditions: types.UnlockConditions{ PublicKeys: []types.SiaPublicKey{ { Algorithm: types.SignatureEd25519, Key: renterPK[:], }, hostSPK, }, SignaturesRequired: 2, }, NewRevisionNumber: fc.RevisionNumber + 1, NewFileSize: fc.FileSize, NewFileMerkleRoot: fc.FileMerkleRoot, NewWindowStart: fc.WindowStart, NewWindowEnd: fc.WindowEnd, NewValidProofOutputs: fc.ValidProofOutputs, NewMissedProofOutputs: fc.MissedProofOutputs, NewUnlockHash: fc.UnlockHash, } // createRevisionSignature will also perform validation on the result, // returning an error if the renter provided an incorrect signature. revisionTransaction, err := createRevisionSignature(noOpRevision, renterRevisionSignature, hostSK, blockHeight) if err != nil { return nil, types.TransactionSignature{}, types.FileContractID{}, err } // Create and add the storage obligation for this file contract. fullTxn, _ := builder.View() so := storageObligation{ SectorRoots: initialSectorRoots, ContractCost: h.settings.MinContractPrice, LockedCollateral: hostCollateral, PotentialStorageRevenue: hostInitialRevenue, RiskedCollateral: hostInitialRisk, OriginTransactionSet: fullTxnSet, RevisionTransactionSet: []types.Transaction{revisionTransaction}, } // Get a lock on the storage obligation. lockErr := h.managedTryLockStorageObligation(so.id()) if lockErr != nil { build.Critical("failed to get a lock on a brand new storage obligation") return nil, types.TransactionSignature{}, types.FileContractID{}, lockErr } defer func() { if err != nil { h.managedUnlockStorageObligation(so.id()) } }() // addStorageObligation will submit the transaction to the transaction // pool, and will only do so if there was not some error in creating the // storage obligation. If the transaction pool returns a consensus // conflict, wait 30 seconds and try again. err = func() error { // Try adding the storage obligation. If there's an error, wait a few // seconds and try again. Eventually time out. It should be noted that // the storage obligation locking is both crappy and incomplete, and // that I'm not sure how this timeout plays with the overall host // timeouts. // // The storage obligation locks should occur at the highest level, not // just when the actual modification is happening. i := 0 for { h.mu.Lock() err = h.addStorageObligation(so) h.mu.Unlock() if err == nil { return nil } if err != nil && i > 4 { h.log.Println(err) builder.Drop() return err } i++ if build.Release == "standard" { time.Sleep(time.Second * 15) } } }() if err != nil { return nil, types.TransactionSignature{}, types.FileContractID{}, err } // Get the host's transaction signatures from the builder. var hostTxnSignatures []types.TransactionSignature _, _, _, txnSigIndices := builder.ViewAdded() for _, sigIndex := range txnSigIndices { hostTxnSignatures = append(hostTxnSignatures, fullTxn.TransactionSignatures[sigIndex]) } return hostTxnSignatures, revisionTransaction.TransactionSignatures[1], so.id(), nil } Sia-1.0.4/modules/host/negotiatedownload.go000066400000000000000000000254761302033070500207220ustar00rootroot00000000000000package host import ( "fmt" "net" "time" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( // errLargeDownloadBatch is returned if the renter requests a download // batch that exceeds the maximum batch size that the host will // accomondate. errLargeDownloadBatch = ErrorCommunication("download request exceeded maximum batch size") // errRequestOutOfBounds is returned when a download request is made which // asks for elements of a sector which do not exist. errRequestOutOfBounds = ErrorCommunication("download request has invalid sector bounds") ) // managedDownloadIteration is responsible for managing a single iteration of // the download loop for RPCDownload. func (h *Host) managedDownloadIteration(conn net.Conn, so *storageObligation) error { // Exchange settings with the renter. err := h.managedRPCSettings(conn) if err != nil { return extendErr("RPCSettings failed: ", err) } // Extend the deadline for the download. conn.SetDeadline(time.Now().Add(modules.NegotiateDownloadTime)) // The renter will either accept or reject the host's settings. err = modules.ReadNegotiationAcceptance(conn) if err == modules.ErrStopResponse { return err // managedRPCDownload will catch this and exit gracefully } else if err != nil { return extendErr("renter rejected host settings: ", ErrorCommunication(err.Error())) } // Grab a set of variables that will be useful later in the function. h.mu.RLock() blockHeight := h.blockHeight secretKey := h.secretKey settings := h.settings h.mu.RUnlock() // Read the download requests, followed by the file contract revision that // pays for them. var requests []modules.DownloadAction var paymentRevision types.FileContractRevision err = encoding.ReadObject(conn, &requests, modules.NegotiateMaxDownloadActionRequestSize) if err != nil { return extendErr("failed to read download requests:", ErrorConnection(err.Error())) } err = encoding.ReadObject(conn, &paymentRevision, modules.NegotiateMaxFileContractRevisionSize) if err != nil { return extendErr("failed to read payment revision:", ErrorConnection(err.Error())) } // Verify that the request is acceptable, and then fetch all of the data // for the renter. existingRevision := so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0] var payload [][]byte err = func() error { // Check that the length of each file is in-bounds, and that the total // size being requested is acceptable. var totalSize uint64 for _, request := range requests { if request.Length > modules.SectorSize || request.Offset+request.Length > modules.SectorSize { return extendErr("download iteration request failed: ", errRequestOutOfBounds) } totalSize += request.Length } if totalSize > settings.MaxDownloadBatchSize { return extendErr("download iteration batch failed: ", errLargeDownloadBatch) } // Verify that the correct amount of money has been moved from the // renter's contract funds to the host's contract funds. expectedTransfer := settings.MinDownloadBandwidthPrice.Mul64(totalSize) err = verifyPaymentRevision(existingRevision, paymentRevision, blockHeight, expectedTransfer) if err != nil { return extendErr("payment verification failed: ", err) } // Load the sectors and build the data payload. for _, request := range requests { sectorData, err := h.ReadSector(request.MerkleRoot) if err != nil { return extendErr("failed to load sector: ", ErrorInternal(err.Error())) } payload = append(payload, sectorData[request.Offset:request.Offset+request.Length]) } return nil }() if err != nil { modules.WriteNegotiationRejection(conn, err) // Error not reported to preserve type in extendErr return extendErr("download request rejected: ", err) } // Revision is acceptable, write acceptance. err = modules.WriteNegotiationAcceptance(conn) if err != nil { return extendErr("failed to write acceptance for renter revision: ", ErrorConnection(err.Error())) } // Renter will send a transaction siganture for the file contract revision. var renterSignature types.TransactionSignature err = encoding.ReadObject(conn, &renterSignature, modules.NegotiateMaxTransactionSignatureSize) if err != nil { return extendErr("failed to read renter signature: ", ErrorConnection(err.Error())) } txn, err := createRevisionSignature(paymentRevision, renterSignature, secretKey, blockHeight) // Update the storage obligation. paymentTransfer := existingRevision.NewValidProofOutputs[0].Value.Sub(paymentRevision.NewValidProofOutputs[0].Value) so.PotentialDownloadRevenue = so.PotentialDownloadRevenue.Add(paymentTransfer) so.RevisionTransactionSet = []types.Transaction{{ FileContractRevisions: []types.FileContractRevision{paymentRevision}, TransactionSignatures: []types.TransactionSignature{renterSignature, txn.TransactionSignatures[1]}, }} err = h.modifyStorageObligation(*so, nil, nil, nil) if err != nil { return extendErr("failed to modify storage obligation: ", ErrorInternal(modules.WriteNegotiationRejection(conn, err).Error())) } // Write acceptance to the renter - the data request can be fulfilled by // the host, the payment is satisfactory, signature is correct. Then send // the host signature and all of the data. err = modules.WriteNegotiationAcceptance(conn) if err != nil { return extendErr("failed to write acceptance following obligation modification: ", ErrorConnection(err.Error())) } err = encoding.WriteObject(conn, txn.TransactionSignatures[1]) if err != nil { return extendErr("failed to write signature: ", ErrorConnection(err.Error())) } err = encoding.WriteObject(conn, payload) if err != nil { return extendErr("failed to write payload: ", ErrorConnection(err.Error())) } return nil } // verifyPaymentRevision verifies that the revision being provided to pay for // the data has transferred the expected amount of money from the renter to the // host. func verifyPaymentRevision(existingRevision, paymentRevision types.FileContractRevision, blockHeight types.BlockHeight, expectedTransfer types.Currency) error { // Check that the revision is well-formed. if len(paymentRevision.NewValidProofOutputs) != 2 || len(paymentRevision.NewMissedProofOutputs) != 3 { return errBadContractOutputCounts } // Check that the time to finalize and submit the file contract revision // has not already passed. if existingRevision.NewWindowStart-revisionSubmissionBuffer <= blockHeight { return errLateRevision } // Determine the amount that was transferred from the renter. if paymentRevision.NewValidProofOutputs[0].Value.Cmp(existingRevision.NewValidProofOutputs[0].Value) > 0 { return extendErr("renter increased its valid proof output: ", errHighRenterValidOutput) } fromRenter := existingRevision.NewValidProofOutputs[0].Value.Sub(paymentRevision.NewValidProofOutputs[0].Value) // Verify that enough money was transferred. if fromRenter.Cmp(expectedTransfer) < 0 { s := fmt.Sprintf("expected at least %v to be exchanged, but %v was exchanged: ", expectedTransfer, fromRenter) return extendErr(s, errHighRenterValidOutput) } // Determine the amount of money that was transferred to the host. if existingRevision.NewValidProofOutputs[1].Value.Cmp(paymentRevision.NewValidProofOutputs[1].Value) > 0 { return extendErr("host valid proof output was decreased: ", errLowHostValidOutput) } toHost := paymentRevision.NewValidProofOutputs[1].Value.Sub(existingRevision.NewValidProofOutputs[1].Value) // Verify that enough money was transferred. if !toHost.Equals(fromRenter) { s := fmt.Sprintf("expected exactly %v to be transferred to the host, but %v was transferred: ", fromRenter, toHost) return extendErr(s, errLowHostValidOutput) } // If the renter's valid proof output is larger than the renter's missed // proof output, the renter has incentive to see the host fail. Make sure // that this incentive is not present. if paymentRevision.NewValidProofOutputs[0].Value.Cmp(paymentRevision.NewMissedProofOutputs[0].Value) > 0 { return extendErr("renter has incentive to see host fail: ", errHighRenterMissedOutput) } // Check that the host is not going to be posting collateral. if paymentRevision.NewMissedProofOutputs[1].Value.Cmp(existingRevision.NewMissedProofOutputs[1].Value) < 0 { collateral := existingRevision.NewMissedProofOutputs[1].Value.Sub(paymentRevision.NewMissedProofOutputs[1].Value) s := fmt.Sprintf("host not expecting to post any collateral, but contract has host posting %v collateral: ", collateral) return extendErr(s, errLowHostMissedOutput) } // Check that the revision count has increased. if paymentRevision.NewRevisionNumber <= existingRevision.NewRevisionNumber { return errBadRevisionNumber } // Check that all of the non-volatile fields are the same. if paymentRevision.ParentID != existingRevision.ParentID { return errBadParentID } if paymentRevision.UnlockConditions.UnlockHash() != existingRevision.UnlockConditions.UnlockHash() { return errBadUnlockConditions } if paymentRevision.NewFileSize != existingRevision.NewFileSize { return errBadFileSize } if paymentRevision.NewFileMerkleRoot != existingRevision.NewFileMerkleRoot { return errBadFileMerkleRoot } if paymentRevision.NewWindowStart != existingRevision.NewWindowStart { return errBadWindowStart } if paymentRevision.NewWindowEnd != existingRevision.NewWindowEnd { return errBadWindowEnd } if paymentRevision.NewUnlockHash != existingRevision.NewUnlockHash { return errBadUnlockHash } if !paymentRevision.NewMissedProofOutputs[1].Value.Equals(existingRevision.NewMissedProofOutputs[1].Value) { return errLowHostMissedOutput } return nil } // managedRPCDownload is responsible for handling an RPC request from the // renter to download data. func (h *Host) managedRPCDownload(conn net.Conn) error { // Get the start time to limit the length of the whole connection. startTime := time.Now() // Perform the file contract revision exchange, giving the renter the most // recent file contract revision and getting the storage obligation that // will be used to pay for the data. _, so, err := h.managedRPCRecentRevision(conn) if err != nil { return extendErr("failed RPCRecentRevision during RPCDownload: ", err) } // The storage obligation is returned with a lock on it. Defer a call to // unlock the storage obligation. defer func() { h.managedUnlockStorageObligation(so.id()) }() // Perform a loop that will allow downloads to happen until the maximum // time for a single connection has been reached. for time.Now().Before(startTime.Add(iteratedConnectionTime)) { err := h.managedDownloadIteration(conn, &so) if err == modules.ErrStopResponse { // The renter has indicated that it has finished downloading the // data, therefore there is no error. Return nil. return nil } else if err != nil { return extendErr("download iteration failed: ", err) } } return nil } Sia-1.0.4/modules/host/negotiateformcontract.go000066400000000000000000000316121302033070500216010ustar00rootroot00000000000000package host import ( "net" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( // errCollateralBudgetExceeded is returned if the host does not have enough // room in the collateral budget to accept a particular file contract. errCollateralBudgetExceeded = ErrorInternal("host has reached its collateral budget and cannot accept the file contract") // errMaxCollateralReached is returned if a file contract is provided which // would require the host to supply more collateral than the host allows // per file contract. errMaxCollateralReached = ErrorInternal("file contract proposal expects the host to pay more than the maximum allowed collateral") ) // contractCollateral returns the amount of collateral that the host is // expected to add to the file contract based on the payout of the file // contract and based on the host settings. func contractCollateral(settings modules.HostInternalSettings, fc types.FileContract) types.Currency { return fc.ValidProofOutputs[1].Value.Sub(settings.MinContractPrice) } // managedAddCollateral adds the host's collateral to the file contract // transaction set, returning the new inputs and outputs that get added to the // transaction, as well as any new parents that get added to the transaction // set. The builder that is used to add the collateral is also returned, // because the new transaction has not yet been signed. func (h *Host) managedAddCollateral(settings modules.HostInternalSettings, txnSet []types.Transaction) (builder modules.TransactionBuilder, newParents []types.Transaction, newInputs []types.SiacoinInput, newOutputs []types.SiacoinOutput, err error) { txn := txnSet[len(txnSet)-1] parents := txnSet[:len(txnSet)-1] fc := txn.FileContracts[0] hostPortion := contractCollateral(settings, fc) builder = h.wallet.RegisterTransaction(txn, parents) err = builder.FundSiacoins(hostPortion) if err != nil { builder.Drop() return nil, nil, nil, nil, extendErr("could not add collateral: ", ErrorInternal(err.Error())) } // Return which inputs and outputs have been added by the collateral call. newParentIndices, newInputIndices, newOutputIndices, _ := builder.ViewAdded() updatedTxn, updatedParents := builder.View() for _, parentIndex := range newParentIndices { newParents = append(newParents, updatedParents[parentIndex]) } for _, inputIndex := range newInputIndices { newInputs = append(newInputs, updatedTxn.SiacoinInputs[inputIndex]) } for _, outputIndex := range newOutputIndices { newOutputs = append(newOutputs, updatedTxn.SiacoinOutputs[outputIndex]) } return builder, newParents, newInputs, newOutputs, nil } // managedRPCFormContract accepts a file contract from a renter, checks the // file contract for compliance with the host settings, and then commits to the // file contract, creating a storage obligation and submitting the contract to // the blockchain. func (h *Host) managedRPCFormContract(conn net.Conn) error { // Send the host settings to the renter. err := h.managedRPCSettings(conn) if err != nil { return extendErr("failed RPCSettings: ", err) } // If the host is not accepting contracts, the connection can be closed. // The renter has been given enough information in the host settings to // understand that the connection is going to be closed. h.mu.RLock() settings := h.settings h.mu.RUnlock() if !settings.AcceptingContracts { h.log.Debugln("Turning down contract because the host is not accepting contracts.") return nil } // Extend the deadline to meet the rest of file contract negotiation. conn.SetDeadline(time.Now().Add(modules.NegotiateFileContractTime)) // The renter will either accept or reject the host's settings. err = modules.ReadNegotiationAcceptance(conn) if err != nil { return extendErr("renter did not accept settings: ", ErrorCommunication(err.Error())) } // If the renter sends an acceptance of the settings, it will be followed // by an unsigned transaction containing funding from the renter and a file // contract which matches what the final file contract should look like. // After the file contract, the renter will send a public key which is the // renter's public key in the unlock conditions that protect the file // contract from revision. var txnSet []types.Transaction var renterPK crypto.PublicKey err = encoding.ReadObject(conn, &txnSet, modules.NegotiateMaxFileContractSetLen) if err != nil { return extendErr("could not read renter transaction set: ", ErrorConnection(err.Error())) } err = encoding.ReadObject(conn, &renterPK, modules.NegotiateMaxSiaPubkeySize) if err != nil { return extendErr("could not read renter public key: ", ErrorConnection(err.Error())) } // The host verifies that the file contract coming over the wire is // acceptable. err = h.managedVerifyNewContract(txnSet, renterPK) if err != nil { // The incoming file contract is not acceptable to the host, indicate // why to the renter. modules.WriteNegotiationRejection(conn, err) // Error ignored to preserve type in extendErr return extendErr("contract verification failed: ", err) } // The host adds collateral to the transaction. txnBuilder, newParents, newInputs, newOutputs, err := h.managedAddCollateral(settings, txnSet) if err != nil { modules.WriteNegotiationRejection(conn, err) // Error ignored to preserve type in extendErr return extendErr("failed to add collateral: ", err) } // The host indicates acceptance, and then sends any new parent // transactions, inputs and outputs that were added to the transaction. err = modules.WriteNegotiationAcceptance(conn) if err != nil { return extendErr("accepting verified contract failed: ", ErrorConnection(err.Error())) } err = encoding.WriteObject(conn, newParents) if err != nil { return extendErr("failed to write new parents: ", ErrorConnection(err.Error())) } err = encoding.WriteObject(conn, newInputs) if err != nil { return extendErr("failed to write new inputs: ", ErrorConnection(err.Error())) } err = encoding.WriteObject(conn, newOutputs) if err != nil { return extendErr("failed to write new outputs: ", ErrorConnection(err.Error())) } // The renter will now send a negotiation response, followed by transaction // signatures for the file contract transaction in the case of acceptance. // The transaction signatures will be followed by another transaction // siganture, to sign a no-op file contract revision. err = modules.ReadNegotiationAcceptance(conn) if err != nil { return extendErr("renter did not accept updated transactions: ", ErrorCommunication(err.Error())) } var renterTxnSignatures []types.TransactionSignature var renterRevisionSignature types.TransactionSignature err = encoding.ReadObject(conn, &renterTxnSignatures, modules.NegotiateMaxTransactionSignaturesSize) if err != nil { return extendErr("could not read renter transaction signatures: ", ErrorConnection(err.Error())) } err = encoding.ReadObject(conn, &renterRevisionSignature, modules.NegotiateMaxTransactionSignatureSize) if err != nil { return extendErr("could not read renter revision signatures: ", ErrorConnection(err.Error())) } // The host adds the renter transaction signatures, then signs the // transaction and submits it to the blockchain, creating a storage // obligation in the process. The host's part is done before anything is // written to the renter, but to give the renter confidence, the host will // send the signatures so that the renter can immediately have the // completed file contract. // // During finalization, the siganture for the revision is also checked, and // signatures for the revision transaction are created. h.mu.RLock() hostCollateral := contractCollateral(h.settings, txnSet[len(txnSet)-1].FileContracts[0]) h.mu.RUnlock() hostTxnSignatures, hostRevisionSignature, newSOID, err := h.managedFinalizeContract(txnBuilder, renterPK, renterTxnSignatures, renterRevisionSignature, nil, hostCollateral, types.ZeroCurrency, types.ZeroCurrency) if err != nil { // The incoming file contract is not acceptable to the host, indicate // why to the renter. modules.WriteNegotiationRejection(conn, err) // Error ignored to preserve type in extendErr return extendErr("contract finalization failed: ", err) } defer h.managedUnlockStorageObligation(newSOID) err = modules.WriteNegotiationAcceptance(conn) if err != nil { return extendErr("failed to write acceptance after contract finalization: ", ErrorConnection(err.Error())) } // The host sends the transaction signatures to the renter, followed by the // revision signature. Negotiation is complete. err = encoding.WriteObject(conn, hostTxnSignatures) if err != nil { return extendErr("failed to write host transaction signatures: ", ErrorConnection(err.Error())) } err = encoding.WriteObject(conn, hostRevisionSignature) if err != nil { return extendErr("failed to write host revision signatures: ", ErrorConnection(err.Error())) } return nil } // managedVerifyNewContract checks that an incoming file contract matches the host's // expectations for a valid contract. func (h *Host) managedVerifyNewContract(txnSet []types.Transaction, renterPK crypto.PublicKey) error { // Check that the transaction set is not empty. if len(txnSet) < 1 { return extendErr("zero-length transaction set: ", errEmptyObject) } // Check that there is a file contract in the txnSet. if len(txnSet[len(txnSet)-1].FileContracts) < 1 { return extendErr("transaction without file contract: ", errEmptyObject) } h.mu.RLock() blockHeight := h.blockHeight lockedStorageCollateral := h.financialMetrics.LockedStorageCollateral publicKey := h.publicKey settings := h.settings unlockHash := h.unlockHash h.mu.RUnlock() fc := txnSet[len(txnSet)-1].FileContracts[0] // A new file contract should have a file size of zero. if fc.FileSize != 0 { return errBadFileSize } if fc.FileMerkleRoot != (crypto.Hash{}) { return errBadFileMerkleRoot } // WindowStart must be at least revisionSubmissionBuffer blocks into the // future. if fc.WindowStart <= blockHeight+revisionSubmissionBuffer { h.log.Debugf("A renter tried to form a contract that had a window start which was too soon. The contract started at %v, the current height is %v, the revisionSubmissionBuffer is %v, and the comparison was %v <= %v\n", fc.WindowStart, blockHeight, revisionSubmissionBuffer, fc.WindowStart, blockHeight+revisionSubmissionBuffer) return errEarlyWindow } // WindowEnd must be at least settings.WindowSize blocks after // WindowStart. if fc.WindowEnd < fc.WindowStart+settings.WindowSize { return errSmallWindow } // WindowEnd must not be more than settings.MaxDuration blocks into the // future. if fc.WindowStart > blockHeight+settings.MaxDuration { return errLongDuration } // ValidProofOutputs shoud have 2 outputs (renter + host) and missed // outputs should have 3 (renter + host + void) if len(fc.ValidProofOutputs) != 2 || len(fc.MissedProofOutputs) != 3 { return errBadContractOutputCounts } // The unlock hashes of the valid and missed proof outputs for the host // must match the host's unlock hash. The third missed output should point // to the void. if fc.ValidProofOutputs[1].UnlockHash != unlockHash || fc.MissedProofOutputs[1].UnlockHash != unlockHash || fc.MissedProofOutputs[2].UnlockHash != (types.UnlockHash{}) { return errBadPayoutUnlockHashes } // Check that the payouts for the valid proof outputs and the missed proof // outputs are the same - this is important because no data has been added // to the file contract yet. if !fc.ValidProofOutputs[1].Value.Equals(fc.MissedProofOutputs[1].Value) { return errMismatchedHostPayouts } // Check that there's enough payout for the host to cover at least the // contract price. This will prevent negative currency panics when working // with the collateral. if fc.ValidProofOutputs[1].Value.Cmp(settings.MinContractPrice) < 0 { return errLowHostValidOutput } // Check that the collateral does not exceed the maximum amount of // collateral allowed. expectedCollateral := contractCollateral(settings, fc) if expectedCollateral.Cmp(settings.MaxCollateral) > 0 { return errMaxCollateralReached } // Check that the host has enough room in the collateral budget to add this // collateral. if lockedStorageCollateral.Add(expectedCollateral).Cmp(settings.CollateralBudget) > 0 { return errCollateralBudgetExceeded } // The unlock hash for the file contract must match the unlock hash that // the host knows how to spend. expectedUH := types.UnlockConditions{ PublicKeys: []types.SiaPublicKey{ { Algorithm: types.SignatureEd25519, Key: renterPK[:], }, publicKey, }, SignaturesRequired: 2, }.UnlockHash() if fc.UnlockHash != expectedUH { return errBadUnlockHash } // Check that the transaction set has enough fees on it to get into the // blockchain. setFee := modules.CalculateFee(txnSet) minFee, _ := h.tpool.FeeEstimation() if setFee.Cmp(minFee) < 0 { return errLowTransactionFees } return nil } Sia-1.0.4/modules/host/negotiaterecentrevision.go000066400000000000000000000144541302033070500221440ustar00rootroot00000000000000package host import ( "crypto/rand" "errors" "net" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) var ( // errRevisionWrongPublicKeyCount is returned when a stored file contract // revision does not have enough public keys - such a situation should // never happen, and is a critical / developer error. errRevisionWrongPublicKeyCount = errors.New("wrong number of public keys in the unlock conditions of the file contract revision") ) // managedVerifyChallengeResponse will verify that the renter's response to the // challenge provided by the host is correct. In the process, the storage // obligation and file contract revision will be loaded and returned. // // The storage obligation is returned under a storage obligation lock. func (h *Host) managedVerifyChallengeResponse(fcid types.FileContractID, challenge crypto.Hash, challengeResponse crypto.Signature) (so storageObligation, recentRevision types.FileContractRevision, revisionSigs []types.TransactionSignature, err error) { // Grab a lock before it is possible to perform any operations on the // storage obligation. Defer a call to unlock in the event of an error. If // there is no error, the storage obligation will be returned with a lock. err = h.managedTryLockStorageObligation(fcid) if err != nil { err = extendErr("could not get "+fcid.String()+" lock: ", ErrorInternal(err.Error())) return storageObligation{}, types.FileContractRevision{}, nil, err } defer func() { if err != nil { h.managedUnlockStorageObligation(fcid) } }() // Fetch the storage obligation, which has the revision, which has the // renter's public key. h.mu.RLock() defer h.mu.RUnlock() err = h.db.View(func(tx *bolt.Tx) error { so, err = getStorageObligation(tx, fcid) return err }) if err != nil { err = extendErr("could not fetch "+fcid.String()+": ", ErrorInternal(err.Error())) return storageObligation{}, types.FileContractRevision{}, nil, err } // Pull out the file contract revision and the revision's signatures from // the transaction. revisionTxn := so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1] recentRevision = revisionTxn.FileContractRevisions[0] for _, sig := range revisionTxn.TransactionSignatures { // Checking for just the parent id is sufficient, an over-signed file // contract is invalid. if sig.ParentID == crypto.Hash(fcid) { revisionSigs = append(revisionSigs, sig) } } // Verify that the challegne response matches the public key. var renterPK crypto.PublicKey // Sanity check - there should be two public keys. if len(recentRevision.UnlockConditions.PublicKeys) != 2 { // The error has to be set here so that the defered error check will // unlock the storage obligation. h.log.Critical("wrong public key count in file contract revision") err = errRevisionWrongPublicKeyCount err = extendErr("wrong public key count for "+fcid.String()+": ", ErrorInternal(err.Error())) return storageObligation{}, types.FileContractRevision{}, nil, err } copy(renterPK[:], recentRevision.UnlockConditions.PublicKeys[0].Key) err = crypto.VerifyHash(challenge, renterPK, challengeResponse) if err != nil { err = extendErr("bad signature from renter: ", ErrorCommunication(err.Error())) return storageObligation{}, types.FileContractRevision{}, nil, err } return so, recentRevision, revisionSigs, nil } // managedRPCRecentRevision sends the most recent known file contract // revision, including signatures, to the renter, for the file contract with // the id given by the renter. // // The storage obligation is returned under a storage obligation lock. func (h *Host) managedRPCRecentRevision(conn net.Conn) (types.FileContractID, storageObligation, error) { // Set the negotiation deadline. conn.SetDeadline(time.Now().Add(modules.NegotiateRecentRevisionTime)) // Receive the file contract id from the renter. var fcid types.FileContractID err := encoding.ReadObject(conn, &fcid, uint64(len(fcid))) if err != nil { return types.FileContractID{}, storageObligation{}, extendErr("could not read file contract id: ", ErrorConnection(err.Error())) } // Send a challenge to the renter to verify that the renter has write // access to the revision being opened. var challenge crypto.Hash _, err = rand.Read(challenge[:]) if err != nil { return types.FileContractID{}, storageObligation{}, ErrorInternal(err.Error()) } err = encoding.WriteObject(conn, challenge) if err != nil { return types.FileContractID{}, storageObligation{}, extendErr("cound not write challenge: ", ErrorConnection(err.Error())) } // Read the signed response from the renter. var challengeResponse crypto.Signature err = encoding.ReadObject(conn, &challengeResponse, uint64(len(challengeResponse))) if err != nil { return types.FileContractID{}, storageObligation{}, extendErr("could not read challenge response: ", ErrorConnection(err.Error())) } // Verify the response. In the process, fetch the related storage // obligation, file contract revision, and transaction signatures. so, recentRevision, revisionSigs, err := h.managedVerifyChallengeResponse(fcid, challenge, challengeResponse) if err != nil { modules.WriteNegotiationRejection(conn, err) // Error not reported to preserve error type in extendErr. return types.FileContractID{}, storageObligation{}, extendErr("challenge failed: ", err) } // Defer a call to unlock the storage obligation in the event of an error. defer func() { if err != nil { h.managedUnlockStorageObligation(fcid) } }() // Send the file contract revision and the corresponding signatures to the // renter. err = modules.WriteNegotiationAcceptance(conn) if err != nil { err = extendErr("failed to write challenge acceptance: ", ErrorConnection(err.Error())) return types.FileContractID{}, storageObligation{}, err } err = encoding.WriteObject(conn, recentRevision) if err != nil { err = extendErr("failed to write recent revision: ", ErrorConnection(err.Error())) return types.FileContractID{}, storageObligation{}, err } err = encoding.WriteObject(conn, revisionSigs) if err != nil { err = extendErr("failed to write recent revision singatures: ", ErrorConnection(err.Error())) return types.FileContractID{}, storageObligation{}, err } return fcid, so, nil } Sia-1.0.4/modules/host/negotiaterenewcontract.go000066400000000000000000000315121302033070500217550ustar00rootroot00000000000000package host import ( "errors" "net" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( // errRenewDoesNotExtend is returned if a file contract renewal is // presented which does not extend the existing file contract. errRenewDoesNotExtend = errors.New("file contract renewal does not extend the existing file contract") ) // renewBaseCollateral returns the base collateral on the storage in the file // contract, using the host's external settings and the starting file contract. func renewBaseCollateral(so storageObligation, settings modules.HostExternalSettings, fc types.FileContract) types.Currency { if fc.WindowEnd <= so.proofDeadline() { return types.NewCurrency64(0) } timeExtension := fc.WindowEnd - so.proofDeadline() return settings.Collateral.Mul64(fc.FileSize).Mul64(uint64(timeExtension)) } // renewBasePrice returns the base cost of the storage in the file contract, // using the host external settings and the starting file contract. func renewBasePrice(so storageObligation, settings modules.HostExternalSettings, fc types.FileContract) types.Currency { if fc.WindowEnd <= so.proofDeadline() { return types.NewCurrency64(0) } timeExtension := fc.WindowEnd - so.proofDeadline() return settings.StoragePrice.Mul64(fc.FileSize).Mul64(uint64(timeExtension)) } // renewContractCollateral returns the amount of collateral that the host is // expected to add to the file contract based on the file contract and host // settings. func renewContractCollateral(so storageObligation, settings modules.HostExternalSettings, fc types.FileContract) types.Currency { return fc.ValidProofOutputs[1].Value.Sub(settings.ContractPrice).Sub(renewBasePrice(so, settings, fc)) } // managedAddRenewCollateral adds the host's collateral to the renewed file // contract. func (h *Host) managedAddRenewCollateral(so storageObligation, settings modules.HostExternalSettings, txnSet []types.Transaction) (builder modules.TransactionBuilder, newParents []types.Transaction, newInputs []types.SiacoinInput, newOutputs []types.SiacoinOutput, err error) { txn := txnSet[len(txnSet)-1] parents := txnSet[:len(txnSet)-1] fc := txn.FileContracts[0] hostPortion := renewContractCollateral(so, settings, fc) builder = h.wallet.RegisterTransaction(txn, parents) err = builder.FundSiacoins(hostPortion) if err != nil { builder.Drop() return nil, nil, nil, nil, extendErr("could not add collateral: ", ErrorInternal(err.Error())) } // Return which inputs and outputs have been added by the collateral call. newParentIndices, newInputIndices, newOutputIndices, _ := builder.ViewAdded() updatedTxn, updatedParents := builder.View() for _, parentIndex := range newParentIndices { newParents = append(newParents, updatedParents[parentIndex]) } for _, inputIndex := range newInputIndices { newInputs = append(newInputs, updatedTxn.SiacoinInputs[inputIndex]) } for _, outputIndex := range newOutputIndices { newOutputs = append(newOutputs, updatedTxn.SiacoinOutputs[outputIndex]) } return builder, newParents, newInputs, newOutputs, nil } // managedRenewContract accepts a request to renew a file contract. func (h *Host) managedRPCRenewContract(conn net.Conn) error { // Perform the recent revision protocol to get the file contract being // revised. _, so, err := h.managedRPCRecentRevision(conn) if err != nil { return extendErr("RPCRecentRevision failed: ", err) } // The storage obligation is received with a lock. Defer a call to unlock // the storage obligation. defer func() { h.managedUnlockStorageObligation(so.id()) }() // Perform the host settings exchange with the renter. err = h.managedRPCSettings(conn) if err != nil { return extendErr("RPCSettings failed: ", err) } // Set the renewal deadline. conn.SetDeadline(time.Now().Add(modules.NegotiateRenewContractTime)) // The renter will either accept or reject the host's settings. err = modules.ReadNegotiationAcceptance(conn) if err != nil { return extendErr("renter rejected the host settings: ", ErrorCommunication(err.Error())) } // If the renter sends an acceptance of the settings, it will be followed // by an unsigned transaction containing funding from the renter and a file // contract which matches what the final file contract should look like. // After the file contract, the renter will send a public key which is the // renter's public key in the unlock conditions that protect the file // contract from revision. var txnSet []types.Transaction var renterPK crypto.PublicKey err = encoding.ReadObject(conn, &txnSet, modules.NegotiateMaxFileContractSetLen) if err != nil { return extendErr("unable to read transaction set: ", ErrorConnection(err.Error())) } err = encoding.ReadObject(conn, &renterPK, modules.NegotiateMaxSiaPubkeySize) if err != nil { return extendErr("unable to read renter public key: ", ErrorConnection(err.Error())) } h.mu.RLock() settings := h.externalSettings() h.mu.RUnlock() // Verify that the transaction coming over the wire is a proper renewal. err = h.managedVerifyRenewedContract(so, txnSet, renterPK) if err != nil { modules.WriteNegotiationRejection(conn, err) // Error is ignored to preserve type for extendErr return extendErr("verification of renewal failed: ", err) } txnBuilder, newParents, newInputs, newOutputs, err := h.managedAddRenewCollateral(so, settings, txnSet) if err != nil { modules.WriteNegotiationRejection(conn, err) // Error is ignored to preserve type for extendErr return extendErr("failed to add collateral: ", err) } // The host indicates acceptance, then sends the new parents, inputs, and // outputs to the transaction. err = modules.WriteNegotiationAcceptance(conn) if err != nil { return extendErr("failed to write acceptance: ", ErrorConnection(err.Error())) } err = encoding.WriteObject(conn, newParents) if err != nil { return extendErr("failed to write new parents: ", ErrorConnection(err.Error())) } err = encoding.WriteObject(conn, newInputs) if err != nil { return extendErr("failed to write new inputs: ", ErrorConnection(err.Error())) } err = encoding.WriteObject(conn, newOutputs) if err != nil { return extendErr("failed to write new outputs: ", ErrorConnection(err.Error())) } // The renter will send a negotiation response, followed by transaction // signatures for the file contract transaction in the case of acceptance. // The transaction signatures will be followed by another transaction // signature to sign the no-op file contract revision associated with the // new file contract. err = modules.ReadNegotiationAcceptance(conn) if err != nil { return extendErr("renter rejected collateral extension: ", ErrorCommunication(err.Error())) } var renterTxnSignatures []types.TransactionSignature var renterRevisionSignature types.TransactionSignature err = encoding.ReadObject(conn, &renterTxnSignatures, modules.NegotiateMaxTransactionSignatureSize) if err != nil { return extendErr("failed to read renter transaction signatures: ", ErrorConnection(err.Error())) } err = encoding.ReadObject(conn, &renterRevisionSignature, modules.NegotiateMaxTransactionSignatureSize) if err != nil { return extendErr("failed to read renter revision signatures: ", ErrorConnection(err.Error())) } // The host adds the renter transaction signatures, then signs the // transaction and submits it to the blockchain, creating a storage // obligation in the process. The host's part is now complete and the // contract is finalized, but to give confidence to the renter the host // will send the sigantures so that the renter can immediately have the // completed file contract. // // During finalization the signatures sent by the renter are all checked. h.mu.RLock() fc := txnSet[len(txnSet)-1].FileContracts[0] renewCollateral := renewContractCollateral(so, settings, fc) renewRevenue := renewBasePrice(so, settings, fc) renewRisk := renewBaseCollateral(so, settings, fc) h.mu.RUnlock() hostTxnSignatures, hostRevisionSignature, newSOID, err := h.managedFinalizeContract(txnBuilder, renterPK, renterTxnSignatures, renterRevisionSignature, so.SectorRoots, renewCollateral, renewRevenue, renewRisk) if err != nil { modules.WriteNegotiationRejection(conn, err) // Error is ignored to preserve type for extendErr return extendErr("failed to finalize contract: ", err) } defer h.managedUnlockStorageObligation(newSOID) err = modules.WriteNegotiationAcceptance(conn) if err != nil { return extendErr("failed to write acceptance: ", ErrorConnection(err.Error())) } // The host sends the transaction signatures to the renter, followed by the // revision signature. Negotiation is complete. err = encoding.WriteObject(conn, hostTxnSignatures) if err != nil { return extendErr("failed to write transaction signatures: ", ErrorConnection(err.Error())) } err = encoding.WriteObject(conn, hostRevisionSignature) if err != nil { return extendErr("failed to write revision signature: ", ErrorConnection(err.Error())) } return nil } // managedVerifyRenewedContract checks that the contract renewal matches the // previous contract and makes all of the appropriate payments. func (h *Host) managedVerifyRenewedContract(so storageObligation, txnSet []types.Transaction, renterPK crypto.PublicKey) error { // Check that the transaction set is not empty. if len(txnSet) < 1 { return extendErr("zero-length transaction set: ", errEmptyObject) } // Check that the transaction set has a file contract. if len(txnSet[len(txnSet)-1].FileContracts) < 1 { return extendErr("transaction without file contract: ", errEmptyObject) } h.mu.RLock() blockHeight := h.blockHeight externalSettings := h.externalSettings() internalSettings := h.settings lockedStorageCollateral := h.financialMetrics.LockedStorageCollateral publicKey := h.publicKey unlockHash := h.unlockHash h.mu.RUnlock() fc := txnSet[len(txnSet)-1].FileContracts[0] // The file size and merkle root must match the file size and merkle root // from the previous file contract. if fc.FileSize != so.fileSize() { return errBadFileSize } if fc.FileMerkleRoot != so.merkleRoot() { return errBadFileMerkleRoot } // The WindowStart must be at least revisionSubmissionBuffer blocks into // the future. if fc.WindowStart <= blockHeight+revisionSubmissionBuffer { return errEarlyWindow } // WindowEnd must be at least settings.WindowSize blocks after WindowStart. if fc.WindowEnd < fc.WindowStart+externalSettings.WindowSize { return errSmallWindow } // ValidProofOutputs shoud have 2 outputs (renter + host) and missed // outputs should have 3 (renter + host + void) if len(fc.ValidProofOutputs) != 2 || len(fc.MissedProofOutputs) != 3 { return errBadContractOutputCounts } // The unlock hashes of the valid and missed proof outputs for the host // must match the host's unlock hash. The third missed output should point // to the void. if fc.ValidProofOutputs[1].UnlockHash != unlockHash || fc.MissedProofOutputs[1].UnlockHash != unlockHash || fc.MissedProofOutputs[2].UnlockHash != (types.UnlockHash{}) { return errBadPayoutUnlockHashes } // Check that the collateral does not exceed the maximum amount of // collateral allowed. expectedCollateral := renewContractCollateral(so, externalSettings, fc) if expectedCollateral.Cmp(externalSettings.MaxCollateral) > 0 { return errMaxCollateralReached } // Check that the host has enough room in the collateral budget to add this // collateral. if lockedStorageCollateral.Add(expectedCollateral).Cmp(internalSettings.CollateralBudget) > 0 { return errCollateralBudgetExceeded } // Check that the missed proof outputs contain enough money, and that the // void output contains enough money. basePrice := renewBasePrice(so, externalSettings, fc) baseCollateral := renewBaseCollateral(so, externalSettings, fc) if fc.ValidProofOutputs[1].Value.Cmp(basePrice.Add(baseCollateral)) < 0 { return errLowHostValidOutput } expectedHostMissedOutput := fc.ValidProofOutputs[1].Value.Sub(basePrice).Sub(baseCollateral) if fc.MissedProofOutputs[1].Value.Cmp(expectedHostMissedOutput) < 0 { return errLowHostMissedOutput } // Check that the void output has the correct value. expectedVoidOutput := basePrice.Add(baseCollateral) if fc.MissedProofOutputs[2].Value.Cmp(expectedVoidOutput) > 0 { return errLowVoidOutput } // The unlock hash for the file contract must match the unlock hash that // the host knows how to spend. expectedUH := types.UnlockConditions{ PublicKeys: []types.SiaPublicKey{ { Algorithm: types.SignatureEd25519, Key: renterPK[:], }, publicKey, }, SignaturesRequired: 2, }.UnlockHash() if fc.UnlockHash != expectedUH { return errBadUnlockHash } // Check that the transaction set has enough fees on it to get into the // blockchain. setFee := modules.CalculateFee(txnSet) minFee, _ := h.tpool.FeeEstimation() if setFee.Cmp(minFee) < 0 { return errLowTransactionFees } return nil } Sia-1.0.4/modules/host/negotiaterevisecontract.go000066400000000000000000000330721302033070500221350ustar00rootroot00000000000000package host import ( "fmt" "net" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // managedRevisionIteration handles one iteration of the revision loop. As a // performance optimization, multiple iterations of revisions are allowed to be // made over the same connection. func (h *Host) managedRevisionIteration(conn net.Conn, so *storageObligation, finalIter bool) error { // Send the settings to the renter. The host will keep going even if it is // not accepting contracts, because in this case the contract already // exists. err := h.managedRPCSettings(conn) if err != nil { return extendErr("RPCSettings failed: ", err) } // Set the negotiation deadline. conn.SetDeadline(time.Now().Add(modules.NegotiateFileContractRevisionTime)) // The renter will either accept or reject the settings + revision // transaction. It may also return a stop response to indicate that it // wishes to terminate the revision loop. err = modules.ReadNegotiationAcceptance(conn) if err == modules.ErrStopResponse { return err // managedRPCReviseContract will catch this and exit gracefully } else if err != nil { return extendErr("renter rejected host settings: ", ErrorCommunication(err.Error())) } // Read some variables from the host for use later in the function. h.mu.RLock() settings := h.settings secretKey := h.secretKey blockHeight := h.blockHeight h.mu.RUnlock() // The renter is going to send its intended modifications, followed by the // file contract revision that pays for them. var modifications []modules.RevisionAction var revision types.FileContractRevision err = encoding.ReadObject(conn, &modifications, settings.MaxReviseBatchSize) if err != nil { return extendErr("unable to read revision modifications: ", ErrorConnection(err.Error())) } err = encoding.ReadObject(conn, &revision, modules.NegotiateMaxFileContractRevisionSize) if err != nil { return extendErr("unable to read proposed revision: ", ErrorConnection(err.Error())) } // First read all of the modifications. Then make the modifications, but // with the ability to reverse them. Then verify the file contract revision // correctly accounts for the changes. var bandwidthRevenue types.Currency // Upload bandwidth. var storageRevenue types.Currency var newCollateral types.Currency var sectorsRemoved []crypto.Hash var sectorsGained []crypto.Hash var gainedSectorData [][]byte err = func() error { for _, modification := range modifications { // Check that the index points to an existing sector root. If the type // is ActionInsert, we permit inserting at the end. if modification.Type == modules.ActionInsert { if modification.SectorIndex > uint64(len(so.SectorRoots)) { return errBadModificationIndex } } else if modification.SectorIndex >= uint64(len(so.SectorRoots)) { return errBadModificationIndex } // Check that the data sent for the sector is not too large. if uint64(len(modification.Data)) > modules.SectorSize { return errLargeSector } switch modification.Type { case modules.ActionDelete: // There is no financial information to change, it is enough to // remove the sector. sectorsRemoved = append(sectorsRemoved, so.SectorRoots[modification.SectorIndex]) so.SectorRoots = append(so.SectorRoots[0:modification.SectorIndex], so.SectorRoots[modification.SectorIndex+1:]...) case modules.ActionInsert: // Check that the sector size is correct. if uint64(len(modification.Data)) != modules.SectorSize { return errBadSectorSize } // Update finances. blocksRemaining := so.proofDeadline() - blockHeight blockBytesCurrency := types.NewCurrency64(uint64(blocksRemaining)).Mul64(modules.SectorSize) bandwidthRevenue = bandwidthRevenue.Add(settings.MinUploadBandwidthPrice.Mul64(modules.SectorSize)) storageRevenue = storageRevenue.Add(settings.MinStoragePrice.Mul(blockBytesCurrency)) newCollateral = newCollateral.Add(settings.Collateral.Mul(blockBytesCurrency)) // Insert the sector into the root list. newRoot := crypto.MerkleRoot(modification.Data) sectorsGained = append(sectorsGained, newRoot) gainedSectorData = append(gainedSectorData, modification.Data) so.SectorRoots = append(so.SectorRoots[:modification.SectorIndex], append([]crypto.Hash{newRoot}, so.SectorRoots[modification.SectorIndex:]...)...) case modules.ActionModify: // Check that the offset and length are okay. Length is already // known to be appropriately small, but the offset needs to be // checked for being appropriately small as well otherwise there is // a risk of overflow. if modification.Offset > modules.SectorSize || modification.Offset+uint64(len(modification.Data)) > modules.SectorSize { return errIllegalOffsetAndLength } // Get the data for the new sector. sector, err := h.ReadSector(so.SectorRoots[modification.SectorIndex]) if err != nil { return extendErr("could not read sector: ", ErrorInternal(err.Error())) } copy(sector[modification.Offset:], modification.Data) // Update finances. bandwidthRevenue = bandwidthRevenue.Add(settings.MinUploadBandwidthPrice.Mul64(uint64(len(modification.Data)))) // Update the sectors removed and gained to indicate that the old // sector has been replaced with a new sector. newRoot := crypto.MerkleRoot(sector) sectorsRemoved = append(sectorsRemoved, so.SectorRoots[modification.SectorIndex]) sectorsGained = append(sectorsGained, newRoot) gainedSectorData = append(gainedSectorData, sector) so.SectorRoots[modification.SectorIndex] = newRoot default: return errUnknownModification } } newRevenue := storageRevenue.Add(bandwidthRevenue) return extendErr("unable to verify updated contract: ", verifyRevision(*so, revision, blockHeight, newRevenue, newCollateral)) }() if err != nil { modules.WriteNegotiationRejection(conn, err) // Error is ignored so that the error type can be preserved in extendErr. return extendErr("rejected proposed modifications: ", err) } // Revision is acceptable, write an acceptance string. err = modules.WriteNegotiationAcceptance(conn) if err != nil { return extendErr("could not accept revision modifications: ", ErrorConnection(err.Error())) } // Renter will send a transaction signature for the file contract revision. var renterSig types.TransactionSignature err = encoding.ReadObject(conn, &renterSig, modules.NegotiateMaxTransactionSignatureSize) if err != nil { return extendErr("could not read renter transaction signature: ", ErrorConnection(err.Error())) } // Verify that the signature is valid and get the host's signature. txn, err := createRevisionSignature(revision, renterSig, secretKey, blockHeight) if err != nil { modules.WriteNegotiationRejection(conn, err) // Error is ignored so that the error type can be preserved in extendErr. return extendErr("could not create revision signature: ", err) } so.PotentialStorageRevenue = so.PotentialStorageRevenue.Add(storageRevenue) so.RiskedCollateral = so.RiskedCollateral.Add(newCollateral) so.PotentialUploadRevenue = so.PotentialUploadRevenue.Add(bandwidthRevenue) so.RevisionTransactionSet = []types.Transaction{txn} err = h.modifyStorageObligation(*so, sectorsRemoved, sectorsGained, gainedSectorData) if err != nil { modules.WriteNegotiationRejection(conn, err) // Error is ignored so that the error type can be preserved in extendErr. return extendErr("could not modify storage obligation: ", ErrorInternal(err.Error())) } // Host will now send acceptance and its signature to the renter. This // iteration is complete. If the finalIter flag is set, StopResponse will // be sent instead. This indicates to the renter that the host wishes to // terminate the revision loop. if finalIter { err = modules.WriteNegotiationStop(conn) } else { err = modules.WriteNegotiationAcceptance(conn) } if err != nil { return extendErr("iteration signal failed to send: ", ErrorConnection(err.Error())) } err = encoding.WriteObject(conn, txn.TransactionSignatures[1]) if err != nil { return extendErr("failed to write revision signatures: ", ErrorConnection(err.Error())) } return nil } // managedRPCReviseContract accepts a request to revise an existing contract. // Revisions can add sectors, delete sectors, and modify existing sectors. func (h *Host) managedRPCReviseContract(conn net.Conn) error { // Set a preliminary deadline for receiving the storage obligation. startTime := time.Now() // Perform the file contract revision exchange, giving the renter the most // recent file contract revision and getting the storage obligation that // will be used to pay for the data. _, so, err := h.managedRPCRecentRevision(conn) if err != nil { return extendErr("RPCRecentRevision failed: ", err) } // The storage obligation is received with a lock on it. Defer a call to // unlock the storage obligation. defer func() { h.managedUnlockStorageObligation(so.id()) }() // Begin the revision loop. The host will process revisions until a // timeout is reached, or until the renter sends a StopResponse. for timeoutReached := false; !timeoutReached; { timeoutReached = time.Since(startTime) > iteratedConnectionTime err := h.managedRevisionIteration(conn, &so, timeoutReached) if err == modules.ErrStopResponse { return nil } else if err != nil { return extendErr("revision iteration failed: ", err) } } return nil } // verifyRevision checks that the revision pays the host correctly, and that // the revision does not attempt any malicious or unexpected changes. func verifyRevision(so storageObligation, revision types.FileContractRevision, blockHeight types.BlockHeight, expectedExchange, expectedCollateral types.Currency) error { // Check that the revision is well-formed. if len(revision.NewValidProofOutputs) != 2 || len(revision.NewMissedProofOutputs) != 3 { return errBadContractOutputCounts } // Check that the time to finalize and submit the file contract revision // has not already passed. if so.expiration()-revisionSubmissionBuffer <= blockHeight { return errLateRevision } oldFCR := so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0] // Check that all non-volatile fields are the same. if oldFCR.ParentID != revision.ParentID { return errBadContractParent } if oldFCR.UnlockConditions.UnlockHash() != revision.UnlockConditions.UnlockHash() { return errBadUnlockConditions } if oldFCR.NewRevisionNumber >= revision.NewRevisionNumber { return errBadRevisionNumber } if revision.NewFileSize != uint64(len(so.SectorRoots))*modules.SectorSize { return errBadFileSize } if oldFCR.NewWindowStart != revision.NewWindowStart { return errBadWindowStart } if oldFCR.NewWindowEnd != revision.NewWindowEnd { return errBadWindowEnd } if oldFCR.NewUnlockHash != revision.NewUnlockHash { return errBadUnlockHash } // Determine the amount that was transferred from the renter. if revision.NewValidProofOutputs[0].Value.Cmp(oldFCR.NewValidProofOutputs[0].Value) > 0 { return extendErr("renter increased its valid proof output: ", errHighRenterValidOutput) } fromRenter := oldFCR.NewValidProofOutputs[0].Value.Sub(revision.NewValidProofOutputs[0].Value) // Verify that enough money was transferred. if fromRenter.Cmp(expectedExchange) < 0 { s := fmt.Sprintf("expected at least %v to be exchanged, but %v was exchanged: ", expectedExchange, fromRenter) return extendErr(s, errHighRenterValidOutput) } // Determine the amount of money that was transferred to the host. if oldFCR.NewValidProofOutputs[1].Value.Cmp(revision.NewValidProofOutputs[1].Value) > 0 { return extendErr("host valid proof output was decreased: ", errLowHostValidOutput) } toHost := revision.NewValidProofOutputs[1].Value.Sub(oldFCR.NewValidProofOutputs[1].Value) // Verify that enough money was transferred. if !toHost.Equals(fromRenter) { s := fmt.Sprintf("expected exactly %v to be transferred to the host, but %v was transferred: ", fromRenter, toHost) return extendErr(s, errLowHostValidOutput) } // If the renter's valid proof output is larger than the renter's missed // proof output, the renter has incentive to see the host fail. Make sure // that this incentive is not present. if revision.NewValidProofOutputs[0].Value.Cmp(revision.NewMissedProofOutputs[0].Value) > 0 { return extendErr("renter has incentive to see host fail: ", errHighRenterMissedOutput) } // Check that the host is not going to be posting more collateral than is // expected. If the new misesd output is greater than the old one, the host // is actually posting negative collateral, which is fine. if revision.NewMissedProofOutputs[1].Value.Cmp(oldFCR.NewMissedProofOutputs[1].Value) <= 0 { collateral := oldFCR.NewMissedProofOutputs[1].Value.Sub(revision.NewMissedProofOutputs[1].Value) if collateral.Cmp(expectedCollateral) > 0 { s := fmt.Sprintf("host expected to post at most %v collateral, but contract has host posting %v: ", expectedCollateral, collateral) return extendErr(s, errLowHostMissedOutput) } } // Check that the revision count has increased. if revision.NewRevisionNumber <= oldFCR.NewRevisionNumber { return errBadRevisionNumber } // The Merkle root is checked last because it is the most expensive check. log2SectorSize := uint64(0) for 1< maximumStorageFolderSize { sm.log.Critical("storage folder", i, "exceeds the maximum allowed storage folder size") return errStorageFolderMaxSizeExceeded } if sf.Size < minimumStorageFolderSize { sm.log.Critical("storage folder", i, "has less than the minimum allowed storage folder size") return errStorageFolderMinSizeViolated } // The amount of storage remaining should not be greater than the // folder size. if sf.SizeRemaining > sf.Size { sm.log.Critical("storage folder", i, "has more storage remaining than it has storage total") return errStorageFolderSizeRemainingDivergence } // The UID has a fixed size. if len(sf.UID) != storageFolderUIDSize { sm.log.Critical("storage folder", i, "has an ID which is not valid") return errStorageFolderInvalidUIDLen } // Check that the storage folder UID is not conflicting with any other // known storage folder UID. conflict, exists := knownUIDs[sf.uidString()] if exists { sm.log.Critical("storage folder", i, "has a duplicate UID, conflicting with storage folder", conflict) return errStorageFolderDuplicateUID } // Add this storage folder's UID to the set of known UIDs. knownUIDs[sf.uidString()] = i } return nil } Sia-1.0.4/modules/host/storagemanager/consts.go000066400000000000000000000121571302033070500215130ustar00rootroot00000000000000package storagemanager import ( "github.com/NebulousLabs/Sia/build" ) const ( // maximumStorageFolders indicates the maximum number of storage folders // that the host allows. Some operations, such as creating a new storage // folder, take longer if there are more storage folders. Static RAM usage // also increases as the number of storage folders increase. For this // reason, a limit on the maximum number of storage folders has been set. maximumStorageFolders = 100 ) var ( // maximumStorageFolderSize sets an upper bound on how large storage // folders in the host are allowed to be. It makes sure that inputs and // constructions are sane. While it's conceivable that someone could create // a rig with a single logical storage folder greater than 128 TiB in size // in production, it's probably not a great idea, especially when you are // allowed to use many storage folders. All told, a single host on today's // constants can support up to ~10 PB of storage. maximumStorageFolderSize = func() uint64 { if build.Release == "dev" { return 1 << 40 // 1 TiB } if build.Release == "standard" { return 1 << 50 // 1 PiB } if build.Release == "testing" { return 1 << 20 // 1 MiB } panic("unrecognized release constant in host - maximum storage folder size") }() // maximumVirtualSectors defines the maximum number of virtual sectors that // can be tied to each physical sector. maximumVirtualSectors = func() int { if build.Release == "dev" { // The testing value is at 35 to provide flexibility. The // development value is at 5 because hitting the virtual sector // limit in a sane development environment is more difficult than // hitting the virtual sector limit in a controlled testing // environment (dev environment doesn't have access to private // methods such as 'addSector'. return 5 } if build.Release == "standard" { // Each virtual sector adds about 8 bytes of load to the host // persistence structures, and additionally adds 8 bytes of load // when reading or modifying a sector. Though a few virtual sectors // with 10e3 or even 100e3 virtual sectors would not be too // detrimental to the host, tens of thousands of physical sectors // that each have ten thousand virtual sectors could pose a problem // for the host. In most situations, a renter will not need more 2 // or 3 virtual sectors when manipulating data, so 250 is generous // as long as the renter is properly encrypting data. 250 is // unlikely to cause the host problems, even if an attacker is // creating hundreds of thousands of physical sectors (an expensive // action!) each with 250 virtual sectors. return 250 } if build.Release == "testing" { return 35 } panic("unrecognized release constant in host - maximum virtual sector size") }() // minimumStorageFolderSize defines the smallest size that a storage folder // is allowed to be. The new design of the storage folder structure means // that this limit is not as relevant as it was originally, but hosts with // little storage capacity are not very useful to the network, and can // actually frustrate price planning. 32 GB has been chosen as a minimum // for the early days of the network, to allow people to experiment in the // beta, but in the future I think something like 256 GB would be much more // appropriate. minimumStorageFolderSize = func() uint64 { if build.Release == "dev" { return 1 << 25 // 32 MiB } if build.Release == "standard" { return 1 << 35 // 32 GiB } if build.Release == "testing" { return 1 << 15 // 32 KiB } panic("unrecognized release constant in host - minimum storage folder size") }() // storageFolderUIDSize determines the number of bytes used to determine // the storage folder UID. Production and development environments use 4 // bytes to minimize the possibility of accidental collisions, and testing // environments use 1 byte so that collisions can be forced while using the // live code. storageFolderUIDSize = func() int { if build.Release == "dev" { return 2 } if build.Release == "standard" { return 4 } if build.Release == "testing" { return 1 } panic("unrecognized release constant in host - storageFolderUIDSize") }() // bucketSectorUsage maps sector IDs to the number of times they are used // in file contracts. If all data is correctly encrypted using a unique // seed, each sector will be in use exactly one time. The host however // cannot control this, and a user may upload unencrypted data or // intentionally upload colliding sectors as a means of attack. The host // can only delete a sector when it is in use zero times. The number of // times a sector is in use is encoded as a big endian uint64. bucketSectorUsage = []byte("BucketSectorUsage") ) // MaximumStorageFolderSize provides the maximumStorageFolderSize value to // other modules for testing purposes. func MaximumStorageFolderSize() uint64 { return maximumStorageFolderSize } // MinimumStorageFolderSize provides the minimumStorageFolderSize value to // other modules for testing purposes. func MinimumStorageFolderSize() uint64 { return minimumStorageFolderSize } Sia-1.0.4/modules/host/storagemanager/dependencies.go000066400000000000000000000110421302033070500226200ustar00rootroot00000000000000package storagemanager import ( "crypto/rand" "errors" "io/ioutil" "os" "strings" "github.com/NebulousLabs/Sia/persist" ) // Fake errors that get returned when a simulated failure of a dependency is // desired for testing. var ( mockErrListen = errors.New("simulated Listen failure") mockErrLoadFile = errors.New("simulated LoadFile failure") mockErrMkdirAll = errors.New("simulated MkdirAll failure") mockErrNewLogger = errors.New("simulated NewLogger failure") mockErrOpenDatabase = errors.New("simulated OpenDatabase failure") mockErrReadFile = errors.New("simulated ReadFile failure") mockErrRemoveFile = errors.New("simulated RemoveFile faulure") mockErrSymlink = errors.New("simulated Symlink failure") mockErrWriteFile = errors.New("simulated WriteFile failure") ) // These interfaces define the StorageManager's dependencies. Mocking // implementation complexity can be reduced by defining each dependency as the // minimum possible subset of the real dependency. type ( // dependencies defines all of the dependencies of the StorageManager. dependencies interface { // loadFile allows the host to load a persistence structure form disk. loadFile(persist.Metadata, interface{}, string) error // mkdirAll gives the host the ability to create chains of folders // within the filesystem. mkdirAll(string, os.FileMode) error // newLogger creates a logger that the host can use to log messages and // write critical statements. newLogger(string) (*persist.Logger, error) // openDatabase creates a database that the host can use to interact // with large volumes of persistent data. openDatabase(persist.Metadata, string) (*persist.BoltDatabase, error) // randRead fills the input bytes with random data. randRead([]byte) (int, error) // readFile reads a file in full from the filesystem. readFile(string) ([]byte, error) // removeFile removes a file from file filesystem. removeFile(string) error // symlink creates a sym link between a source and a destination. symlink(s1, s2 string) error // writeFile writes data to the filesystem using the provided filename. writeFile(string, []byte, os.FileMode) error } ) type ( // productionDependencies is an empty struct that implements all of the // dependencies using full featured libraries. productionDependencies struct{} ) // composeErrors will take two errors and compose them into a single errors // with a longer message. Any nil errors used as inputs will be stripped out, // and if there are zero non-nil inputs then 'nil' will be returned. func composeErrors(errs ...error) error { // Strip out any nil errors. var errStrings []string for _, err := range errs { if err != nil { errStrings = append(errStrings, err.Error()) } } // Return nil if there are no non-nil errors in the input. if len(errStrings) <= 0 { return nil } // Combine all of the non-nil errors into one larger return value. return errors.New(strings.Join(errStrings, "; ")) } // loadFile allows the host to load a persistence structure form disk. func (productionDependencies) loadFile(m persist.Metadata, i interface{}, s string) error { return persist.LoadFile(m, i, s) } // mkdirAll gives the host the ability to create chains of folders within the // filesystem. func (productionDependencies) mkdirAll(s string, fm os.FileMode) error { return os.MkdirAll(s, fm) } // newLogger creates a logger that the host can use to log messages and write // critical statements. func (productionDependencies) newLogger(s string) (*persist.Logger, error) { return persist.NewFileLogger(s) } // openDatabase creates a database that the host can use to interact with large // volumes of persistent data. func (productionDependencies) openDatabase(m persist.Metadata, s string) (*persist.BoltDatabase, error) { return persist.OpenDatabase(m, s) } // randRead fills the input bytes with random data. func (productionDependencies) randRead(b []byte) (int, error) { return rand.Read(b) } // readFile reads a file from the filesystem. func (productionDependencies) readFile(s string) ([]byte, error) { return ioutil.ReadFile(s) } // removeFile removes a file from the filesystem. func (productionDependencies) removeFile(s string) error { return os.Remove(s) } // symlink creates a symlink between a source and a destination file. func (productionDependencies) symlink(s1, s2 string) error { return os.Symlink(s1, s2) } // writeFile writes a file to the filesystem. func (productionDependencies) writeFile(s string, b []byte, fm os.FileMode) error { return ioutil.WriteFile(s, b, fm) } Sia-1.0.4/modules/host/storagemanager/persist.go000066400000000000000000000046101302033070500216660ustar00rootroot00000000000000package storagemanager import ( "crypto/rand" "os" "path/filepath" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/bolt" ) // persistence is the data from the storage manager that gets saved to disk. type persistence struct { SectorSalt crypto.Hash StorageFolders []*storageFolder } // persistData returns the data in the StorageManager that will be saved to // disk. func (sm *StorageManager) persistData() persistence { return persistence{ SectorSalt: sm.sectorSalt, StorageFolders: sm.storageFolders, } } // establishDefaults configures the default settings for the storage manager, // overwriting any existing settings. func (sm *StorageManager) establishDefaults() error { _, err := rand.Read(sm.sectorSalt[:]) if err != nil { return err } // If the sector salt is lost, the storage manager is not going to be able // to figure out where sectors are stored on disk. return sm.saveSync() } // initDB will check that the database has been initialized and if not, will // initialize the database. func (sm *StorageManager) initDB() error { return sm.db.Update(func(tx *bolt.Tx) error { // The storage obligation bucket does not exist, which means the // database needs to be initialized. Create the database buckets. buckets := [][]byte{ bucketSectorUsage, } for _, bucket := range buckets { _, err := tx.CreateBucketIfNotExists(bucket) if err != nil { return err } } return nil }) } // load extracts the saved data from disk and applies it to the storage // manager. func (sm *StorageManager) load() error { p := new(persistence) err := sm.dependencies.loadFile(persistMetadata, p, filepath.Join(sm.persistDir, settingsFile)) if os.IsNotExist(err) { // There is no host.json file, set up sane defaults. return sm.establishDefaults() } else if err != nil { return err } sm.sectorSalt = p.SectorSalt sm.storageFolders = p.StorageFolders return nil } // save stores all of the persistent data of the storage manager to disk. func (sm *StorageManager) save() error { return persist.SaveFile(persistMetadata, sm.persistData(), filepath.Join(sm.persistDir, settingsFile)) } // save stores all of the persistent data of the storage manager to disk. func (sm *StorageManager) saveSync() error { return persist.SaveFileSync(persistMetadata, sm.persistData(), filepath.Join(sm.persistDir, settingsFile)) } Sia-1.0.4/modules/host/storagemanager/sector.go000066400000000000000000000434111302033070500214760ustar00rootroot00000000000000package storagemanager import ( "bytes" "encoding/base64" "encoding/hex" "encoding/json" "errors" "io/ioutil" "path/filepath" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // TODO: Write a sector consistency check - every sector in the host database // should be represented by a sector on disk, and vice-versa. This is closer to // a testing check, because the host is tolerant of disk corruption - it is // okay for there to be information in the sector usage struct that cannot be // retrieved from the disk. The consistency check should return information on // how much corruption there is and what shape it takes. If there are files // found on disk that are not represented in the usage struct, those files // should be reported as well. The consistency check should be accompanied by a // 'purge' mode (perhaps multiple modes) which will delete any files in the // storage folders which are not represented in the sector usage database. // // A similar check should exist for verifying that the host has the correct // folder structure. All of the standard files, plus all of the storage // folders, nothing more. This check belongs in storagefolders.go. // // A final check, the obligations check, should verify that every sector in the // sector usage database is represented correctly by the storage obligations, // and that every sector in the storage obligations is represented by the // sector usage database. // // Disk inconsistencies should be handled by returning errors when trying to // read from the filesystem, which means the problem manifests at the lowest // level, the sector level. Because data is missing, there is no 'repair' // operation that can be supported. The sector usage database should match the // storage obligation database, and should be patched if there's a mismatch. // The storage obligation database gets preference. Any missing sectors will be // treated as if they were filesystem problems. // // The consistency check should be wary of 'SizeRemaining' when it is trying to // do cleanup - if sector removals fail, SizeRemaining should not update as // though the sectors are gone (but should also be correct such that it's the // size of the real sectors + the size of the unremovable files - calculated, // not relative) // TODO: Write an RPC that lets the host share which sectors it has lost. // TODO: Make sure the host will not stutter if it needs to perform operations // on sectors that have been manually deleted. var ( // ErrSectorNotFound is returned when a lookup for a sector fails. ErrSectorNotFound = errors.New("could not find the desired sector") // errDiskTrouble is returned when the host is supposed to have enough // storage to hold a new sector but failures that are likely related to the // disk have prevented the host from successfully adding the sector. errDiskTrouble = errors.New("host unable to add sector despite having the storage capacity to do so") // errInsufficientStorageForSector is returned if the host tries to add a // sector when there is not enough storage remaining on the host to accept // the sector. // // Ideally, the host will adjust pricing as the host starts to fill up, so // this error should be pretty rare. Demand should drive the price up // faster than the Host runs out of space, such that the host is always // hovering around 95% capacity and rarely over 98% or under 90% capacity. errInsufficientStorageForSector = errors.New("not enough storage remaining to accept sector") // errNonVirtualSectorInBatch is thrown if a batch-add for sectors has a // physical sector in it. errNonVirtualSectorInBatch = errors.New("sector added in batch which is not a virtual sector") // errMaxVirtualSectors is returned when a sector cannot be added because // the maximum number of virtual sectors for that sector id already exist. errMaxVirtualSectors = errors.New("sector collides with a physical sector that already has the maximum allowed number of virtual sectors") ) // sectorUsage indicates how a sector is being used. Each block height // represents a point at which a file contract using the sector expires. File // contracts that use the sector multiple times will have their block height // appear multiple times. This data allows the host to figure out what types of // discounts can be applied to data that is reusing sectors. This is primarily // useful for file contract renewals, and really shouldn't be used otherwise. // // The StorageFolder field indicates which storage folder is housing the // sector. type sectorUsage struct { Corrupted bool // If the corrupted flag is set, it means the sector is permanently unreachable. Expiry []types.BlockHeight StorageFolder []byte } // sectorID returns the id that should be used when referring to a sector. // There are lots of sectors, and to minimize their footprint a reduced size // hash is used. Hashes are typically 256 bits to provide collision resistance // against an attacker that is able to peform an obscene number of trials per // second on each of an obscene number of machines. Potential collisions for // sectors are limited because hosts have secret data that the attacker does // not know which is used to salt the transformation of a sector hash to a // sectorID. As a result, an attacker is limited in the number of times they // can try to cause a collision - one random shot every time they upload a // sector, and the attacker has limited ability to learn of the success of the // attempt. Uploads are very slow; even on fast machines there will be less // than 1000 per second. It is therefore safe to reduce the security from // 256 bits to 96 bits, which has a collision resistance of 2^48. A reasonable // upper bound for the number of sectors on a host is 2^32, corresponding with // 16 PB of data. // // 12 bytes can be represented as a filepath using 16 base64 characters. This // keeps the filesize small and therefore limits the amount of load placed on // the filesystem when trying to manage hundreds of thousands or even tens of // millions of sectors in a single folder. func (sm *StorageManager) sectorID(sectorRootBytes []byte) []byte { saltedRoot := crypto.HashAll(sectorRootBytes, sm.sectorSalt) id := make([]byte, base64.RawURLEncoding.EncodedLen(12)) base64.RawURLEncoding.Encode(id, saltedRoot[:12]) return id } // AddSector will add a data sector to the host, correctly selecting the // storage folder in which the sector belongs. func (sm *StorageManager) AddSector(sectorRoot crypto.Hash, expiryHeight types.BlockHeight, sectorData []byte) error { sm.mu.Lock() defer sm.mu.Unlock() // Check that there is enough room for the sector in at least one storage // folder - check will also guarantee that there is at least one storage folder. enoughRoom := false for _, sf := range sm.storageFolders { if sf.SizeRemaining >= modules.SectorSize { enoughRoom = true } } // Determine which storage folder is going to receive the new sector. err := sm.db.Update(func(tx *bolt.Tx) error { // Check whether the sector is a virtual sector. sectorKey := sm.sectorID(sectorRoot[:]) bsu := tx.Bucket(bucketSectorUsage) usageBytes := bsu.Get(sectorKey) var usage sectorUsage if usageBytes != nil { // usageBytes != nil indicates that this sector is already a in the // database, meaning it's a virtual sector. Add the expiration // height to the list of expirations, and then return. err := json.Unmarshal(usageBytes, &usage) if err != nil { return err } // If the sector already has the maximum number of virtual sectors, // return an error. The host handles virtual sectors differently // from physical sectors and therefore needs to limit the number of // times that the same data can be uploaded to the host. For // renters that are properly using encryption and are using // sane/reasonable file contract renewal practices, this limit will // never be reached (sane behavior will cause 3-5 at an absolute // maximum, but the limit is substantially higher). if len(usage.Expiry) >= maximumVirtualSectors { return errMaxVirtualSectors } usage.Expiry = append(usage.Expiry, expiryHeight) usageBytes, err = json.Marshal(usage) if err != nil { return err } return bsu.Put(sm.sectorID(sectorRoot[:]), usageBytes) } // Given that this is a physical sector and not a virtual sector, there // needs to be enough room in the storage folder to accept the sector. if !enoughRoom { return errInsufficientStorageForSector } // Sanity check - sector should have modules.SectorSize bytes. This // sanity check is only important if the sector is not a virtual // sector. if uint64(len(sectorData)) != modules.SectorSize { sm.log.Critical("incorrectly sized sector passed to AddSector in the storage manager") return errors.New("incorrectly sized sector passed to AddSector in the storage manager") } // Try adding the sector to disk. In the event of a failure, the host // will try the next storage folder until there is either a success or // until all options have been exhausted. potentialFolders := sm.storageFolders emptiestFolder, emptiestIndex := emptiestStorageFolder(potentialFolders) for emptiestFolder != nil { sectorPath := filepath.Join(sm.persistDir, emptiestFolder.uidString(), string(sectorKey)) err := sm.dependencies.writeFile(sectorPath, sectorData, 0400) if err != nil { // Log the error. sm.log.Println("Unable to accept sector", err, "into folder", emptiestFolder.uidString()) // Indicate to the user that the storage folder is having write // trouble. emptiestFolder.FailedWrites++ // Remove the attempted write - an an incomplete write can // leave a partial file on disk. Error is not checked, we // already know the disk is having trouble. _ = sm.dependencies.removeFile(sectorPath) // Remove the failed folder from the list of folders that can // be tried. potentialFolders = append(potentialFolders[0:emptiestIndex], potentialFolders[emptiestIndex+1:]...) // Try the next folder. emptiestFolder, emptiestIndex = emptiestStorageFolder(potentialFolders) continue } emptiestFolder.SuccessfulWrites++ // File write succeeded - add the sector to the sector usage // database and return. usage := sectorUsage{ Expiry: []types.BlockHeight{expiryHeight}, StorageFolder: emptiestFolder.UID, } emptiestFolder.SizeRemaining -= modules.SectorSize usageBytes, err = json.Marshal(usage) if err != nil { return err } return bsu.Put(sectorKey, usageBytes) } // There is at least one disk that has room, but the write operation // has failed. return errDiskTrouble }) if err != nil { return err } return sm.save() } // AddSectorBatch will add a set of sectors to the host, correctly selecting // the storage folders in which the sectors belong. The sectors should all be // virtual sectors. func (sm *StorageManager) AddSectorBatch(sectorRoots []crypto.Hash, expiryHeight types.BlockHeight) error { sm.mu.Lock() defer sm.mu.Unlock() err := sm.db.Update(func(tx *bolt.Tx) error { for _, root := range sectorRoots { // Verify that the sector is virtual. sectorKey := sm.sectorID(root[:]) bsu := tx.Bucket(bucketSectorUsage) usageBytes := bsu.Get(sectorKey) var usage sectorUsage if usageBytes == nil { // usageBytes == nil means that this is a new sector, therefore not // a virtual sector, and therefore incompatible with // 'AddSectorBatch'. return errNonVirtualSectorInBatch } err := json.Unmarshal(usageBytes, &usage) if err != nil { return err } // If the sector already has the maximum number of virtual sectors, // return an error. The host handles virtual sectors differently // from physical sectors and therefore needs to limit the number of // times that the same data can be uploaded to the host. For // renters that are properly using encryption and are using // sane/reasonable file contract renewal practices, this limit will // never be reached (sane behavior will cause 3-5 at an absolute // maximum, but the limit is substantially higher). if len(usage.Expiry) >= maximumVirtualSectors { return errMaxVirtualSectors } usage.Expiry = append(usage.Expiry, expiryHeight) usageBytes, err = json.Marshal(usage) if err != nil { return err } err = bsu.Put(sm.sectorID(root[:]), usageBytes) if err != nil { return err } } return nil }) if err != nil { return err } return sm.save() } // ReadSector will pull a sector from disk into memory. func (sm *StorageManager) ReadSector(sectorRoot crypto.Hash) (sectorBytes []byte, err error) { sm.mu.Lock() defer sm.mu.Unlock() err = sm.db.View(func(tx *bolt.Tx) error { bsu := tx.Bucket(bucketSectorUsage) sectorKey := sm.sectorID(sectorRoot[:]) sectorUsageBytes := bsu.Get(sectorKey) if sectorUsageBytes == nil { return ErrSectorNotFound } var su sectorUsage err = json.Unmarshal(sectorUsageBytes, &su) if err != nil { return err } sectorPath := filepath.Join(sm.persistDir, hex.EncodeToString(su.StorageFolder), string(sectorKey)) sectorBytes, err = ioutil.ReadFile(sectorPath) sf := sm.storageFolder(su.StorageFolder) if err != nil { // Mark the read failure in the sector. sf.FailedReads++ return err } sf.SuccessfulReads++ return nil }) return } // RemoveSector will remove a sector from the host at the given expiry height. // If the provided sector does not have an expiration at the given height, an // error will be thrown. func (sm *StorageManager) RemoveSector(sectorRoot crypto.Hash, expiryHeight types.BlockHeight) error { sm.mu.Lock() defer sm.mu.Unlock() return sm.db.Update(func(tx *bolt.Tx) error { // Grab the existing sector usage information from the database. bsu := tx.Bucket(bucketSectorUsage) sectorKey := sm.sectorID(sectorRoot[:]) sectorUsageBytes := bsu.Get(sectorKey) if sectorUsageBytes == nil { return ErrSectorNotFound } var usage sectorUsage err := json.Unmarshal(sectorUsageBytes, &usage) if err != nil { return err } if len(usage.Expiry) == 0 { sm.log.Critical("sector recorded in database, but has no expirations") return ErrSectorNotFound } if len(usage.Expiry) == 1 && usage.Expiry[0] != expiryHeight { return ErrSectorNotFound } // If there are multiple entries in the usage expiry, it means that the // physcial data is in use by other sectors ('virtual sectors'). This // sector can be removed from the usage expiry, but the physical data // needs to remain. if len(usage.Expiry) > 1 { // Find any single entry in the usage that's at the expiry height // and remove it. var i int found := false for i = 0; i < len(usage.Expiry); i++ { if usage.Expiry[i] == expiryHeight { found = true break } } if !found { return ErrSectorNotFound } usage.Expiry = append(usage.Expiry[0:i], usage.Expiry[i+1:]...) // Update the database with the new usage expiry. sectorUsageBytes, err = json.Marshal(usage) if err != nil { return err } return bsu.Put(sectorKey, sectorUsageBytes) } // Get the storage folder that contains the physical sector. var folder *storageFolder for _, sf := range sm.storageFolders { if bytes.Equal(sf.UID, usage.StorageFolder) { folder = sf } } // Remove the sector from the physical disk and update the storage // folder metadata. sectorPath := filepath.Join(sm.persistDir, hex.EncodeToString(usage.StorageFolder), string(sectorKey)) err = sm.dependencies.removeFile(sectorPath) if err != nil { // Indicate that the storage folder is having write troubles. folder.FailedWrites++ return err } folder.SizeRemaining += modules.SectorSize folder.SuccessfulWrites++ err = sm.save() if err != nil { return err } // Delete the sector from the bucket - there are no more instances of // this sector in the host. return bsu.Delete(sm.sectorID(sectorRoot[:])) }) } // DeleteSector deletes a sector from the host explicitly, meaning that the // host will be unable to transfer that sector to a renter, and that the host // will be unable to perform a storage proof on that sector. This function is // not intended to be used, however is available so that hosts can easily // comply if compelled by their government to delete certain data. func (sm *StorageManager) DeleteSector(sectorRoot crypto.Hash) error { sm.mu.Lock() defer sm.mu.Unlock() sm.resourceLock.RLock() defer sm.resourceLock.RUnlock() if sm.closed { return errStorageManagerClosed } return sm.db.Update(func(tx *bolt.Tx) error { // Check that the sector exists in the database. bsu := tx.Bucket(bucketSectorUsage) sectorKey := sm.sectorID(sectorRoot[:]) sectorUsageBytes := bsu.Get(sectorKey) if sectorUsageBytes == nil { return ErrSectorNotFound } var usage sectorUsage err := json.Unmarshal(sectorUsageBytes, &usage) if err != nil { return err } // Get the storage folder that contains the physical sector. var folder *storageFolder for _, sf := range sm.storageFolders { if bytes.Equal(sf.UID, usage.StorageFolder) { folder = sf } } // Remove the sector from the physical disk and update the storage // folder metadata. The file is removed from disk as early as possible // to prevent potential errors from stopping the delete. sectorPath := filepath.Join(sm.persistDir, hex.EncodeToString(usage.StorageFolder), string(sectorKey)) err = sm.dependencies.removeFile(sectorPath) if err != nil { // Indicate that the storage folder is having write troubles. folder.FailedWrites++ return err } folder.SizeRemaining += modules.SectorSize folder.SuccessfulWrites++ err = sm.save() if err != nil { return err } // After removing the file from disk, remove the file from the // database. return bsu.Delete(sectorKey) }) } Sia-1.0.4/modules/host/storagemanager/sector_test.go000066400000000000000000000036551302033070500225430ustar00rootroot00000000000000package storagemanager import ( "testing" "github.com/NebulousLabs/Sia/types" ) // TestMaxVirtualSectors checks that the max virtual sector limit is enforced // when adding sectors. func TestMaxVirtualSectors(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() smt, err := newStorageManagerTester("TestMaxVirtualSectors") if err != nil { t.Fatal(err) } defer smt.Close() // Add a storage folder to receive a sector. err = smt.sm.AddStorageFolder(smt.persistDir, minimumStorageFolderSize) if err != nil { t.Fatal(err) } // Add the first instance of the sector. sectorRoot, sectorData, err := createSector() if err != nil { t.Fatal(err) } err = smt.sm.AddSector(sectorRoot, 1, sectorData) if err != nil { t.Fatal(err) } // Add virtual instances of the sector until there are no more available // virual slots. for i := 1; i < maximumVirtualSectors; i++ { err = smt.sm.AddSector(sectorRoot, types.BlockHeight(i%3+2), sectorData) if err != nil { t.Fatal(err) } } // Add another virtual sector, an error should be returned. err = smt.sm.AddSector(sectorRoot, 1, sectorData) if err != errMaxVirtualSectors { t.Fatal(err) } } // TestBadSectorAdd triggers a panic by trying to add an illegal sector. func TestBadSectorAdd(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() smt, err := newStorageManagerTester("TestBadSectorAdd") if err != nil { t.Fatal(err) } defer smt.Close() // Add a storage folder to receive a sector. err = smt.sm.AddStorageFolder(smt.persistDir, minimumStorageFolderSize) if err != nil { t.Fatal(err) } defer func() { r := recover() if r == nil { t.Fatal("did not trigger panic when adding a bad sector") } }() sectorRoot, sectorData, err := createSector() if err != nil { t.Fatal(err) } // Error doesn't need to be checked, a panic will be thrown. _ = smt.sm.AddSector(sectorRoot, 1, sectorData[:1]) t.Fatal("panic not thrown") } Sia-1.0.4/modules/host/storagemanager/storagefolders.go000066400000000000000000000572231302033070500232300ustar00rootroot00000000000000package storagemanager // storagefolder.go is responsible for managing the storage folders within the // host. Storage folders can be added, resized, or removed. There are several // features in place to make sure that the host is always using a reasonable // amount of resources. Sectors in the host are currently always 4 MiB (in the // standard release), though support for different sizes is planned. Because of // the reliance on the cached Merkle trees, sector sizes are likely to always // be a power of 2. // // Though storage folders each contain a bunch of sectors, there is no mapping // from a storage folder to the sectors that it contains. Instead, one must // either look at the filesystem or go through the sector usage database. // There is a mapping from a sector to the storage folder that it is in, so a // list of sectors for each storage folder can be obtained, though the // operation is expensive. It is not recommended that you try to look at the // filesystem to see all of the sectors in a storage folder, because all of the // golang implementations that let you do this load the whole directory into // memory at once, and these directories may contain millions of sectors. // // Strict resource limits are maintained, to make sure that any user behavior // that would strain the host will return an error instead of causing the user // problems. The number of storage folders is capped, the allowed size for a // storage folder has a range, and anything else that might have a linear or // nonconstant effect on resource consumption is capped. // // Sectors are meant to be spread out across the storage folders as evenly as // possible, but this is done in a very passive way. When a storage folder is // added, sectors are not moved from the other storage folder to optimize for a // quick operation. When a storage folder is reduced in size, sectors are only // moved if there is not enough room on the remainder of the storage folder to // hold all of the sectors. // // Storage folders are identified by an ID. This ID is short (4 bytes) and is // randomly generated but is guaranteed not to conflict with any other storage // folder IDs (if a conflict is generated randomly, a new random folder is // chosen). A counter was rejected because storage folders can be removed and // added arbitrarily, and there should be a firm difference between accessing a // storage folder by index vs. accessing a storage folder by ID. // // Storage folders statically track how much of their storage is unused. // Because there is no mapping from a storage folder to the sectors that it // contains, a static mapping must be manually maintained. While it would be // possible to track which sectors are in each storage folder by using nested // buckets in the sector usage database, the implementation cost is high, and // is perceived to be higher than the implementation cost of statically // tracking the amount of storage remaining. Also the introduction of nested // buckets relies on fancier, less used features in the boltdb dependency, // which carries a higher error risk. // TODO: Need to add some command to 'siad' that will correctly repoint a // storage folder to a new mountpoint. As best I can tell, this needs to happen // while siad is not running. Either that, or 'siac' needs to do the whole // shutdown thing itself? Still unclear. import ( "bytes" "encoding/hex" "encoding/json" "errors" "fmt" "os" "path/filepath" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/bolt" ) var ( // ErrEmptyPath is returned if a path is an empty string or was not // provided. ErrEmptyPath = errors.New("must provide nonempty path string for storage folder") // ErrLargeStorageFolder is returned if a new storage folder or a resized // storage folder would exceed the maximum allowed size. ErrLargeStorageFolder = fmt.Errorf("maximum allowed size for a storage folder is %v bytes", maximumStorageFolderSize) // ErrNoResize is returned if a new size is provided for a storage folder // that is the same as the current size of the storage folder. ErrNoResize = errors.New("storage folder selected for resize, but new size is same as current size") // ErrSmallStorageFolder is returned if a new storage folder is not large // enough to meet the requirements for the minimum storage folder size. ErrSmallStorageFolder = fmt.Errorf("minimum allowed size for a storage folder is %v bytes", minimumStorageFolderSize) // ErrIncompleteOffload is returned when the host is tasked with offloading // sectors from a storage folder but is unable to offload the requested // number - but is able to offload some of them. ErrIncompleteOffload = errors.New("could not successfully offload specified number of sectors from storage folder") // ErrRelativePath is returned if a path must be absolute. ErrRelativePath = errors.New("storage folder paths must be absolute") // ErrRepeatFolder is returned if a storage folder is added which links to // a path that is already in use by another storage folder. Only exact path // matches will trigger the error. ErrRepeatFolder = errors.New("selected path is already in use as a storage folder, please use 'resize'") // errBadStorageFolderIndex is returned if a storage folder is requested // that does not have the correct index. errBadStorageFolderIndex = errors.New("no storage folder exists at that index") // errInsufficientRemainingStorageForRemoval is returned if the remaining // storage folders do not have enough space remaining to support being // removed. errInsufficientRemainingStorageForRemoval = errors.New("not enough storage remaining to support removal of disk") // errInsufficientRemainingStorageForShrink is returned if the remaining // storage folders do not have enough space remaining to support being // reduced in size. errInsufficientRemainingStorageForShrink = errors.New("not enough storage remaining to support shrinking of disk") // errMaxStorageFolders indicates that the limit on the number of allowed // storage folders has been reached. errMaxStorageFolders = fmt.Errorf("host can only accept up to %v storage folders", maximumStorageFolders) // errStorageFolderNotFolder is returned if a storage folder gets added // that is not a folder. errStorageFolderNotFolder = errors.New("must use an existing folder") ) // storageFolder tracks a folder that is being used to store sectors. There is // a corresponding symlink in the host directory that points to whatever folder // the user has chosen for storing data (usually, a separate drive will be // mounted at that point). // // 'Size' is set by the user, indicating how much data can be placed into that // folder before the host should consider it full. Size is measured in bytes, // but only accounts for the actual raw data. Sia also places a nontrivial // amount of load on the filesystem, potentially to the tune of millions of // files. These files have long, cryptographic names and may take up as much as // a gigabyte of space in filesystem overhead, depending on how the filesystem // is architected. The host is programmed to gracefully handle full disks, so // while it might cause the user surprise that the host can't break past 99% // utilization, there should not be any issues if the user overestimates how // much storage is available in the folder they have offered to Sia. The host // will put the drive at 100% utilization, which may cause performance // slowdowns or other errors if non-Sia programs are also trying to use the // filesystem. If users are experiencing problems, having them set the storage // folder size to 98% of the actual drive size is probably going to fix most of // the issues. // // 'SizeRemaining' is a variable that remembers how much storage is remaining // in the storage folder. It is managed manually, and is updated every time a // sector is added to or removed from the storage folder. Because there is no // property that inherently guarantees the correctness of 'SizeRemaining', // implementation must be careful to maintain consistency. // // The UID of the storage folder is a small number of bytes that uniquely // identify the storage folder. The UID is generated randomly, but in such a // way as to guarantee that it will not collide with the ids of other storage // folders. The UID is used (via the uidString function) to determine the name // of the symlink which points to the folder holding the data for this storage // folder. // // Statistics are kept on the integrity of reads and writes. Ideally, the // filesystem is never returning errors, but if errors are being returned they // will be tracked and can be reported to the user. type storageFolder struct { Path string UID []byte Size uint64 // bytes SizeRemaining uint64 // bytes FailedReads uint64 FailedWrites uint64 SuccessfulReads uint64 SuccessfulWrites uint64 } // emptiestStorageFolder takes a set of storage folders and returns the storage // folder with the lowest utilization by percentage. 'nil' is returned if there // are no storage folders provided with sufficient free space for a sector. // // Refusing to return a storage folder that does not have enough space prevents // the host from overfilling a storage folder. func emptiestStorageFolder(sfs []*storageFolder) (*storageFolder, int) { mostFree := float64(-1) // Set lower than the min amount available to protect from floating point imprecision. winningIndex := -1 // Set to impossible value to prevent unintentionally returning the wrong storage folder. winner := false for i, sf := range sfs { // Check that this storage folder has at least enough space to hold a // new sector. Also perform a sanity check that the storage folder has // a sane amount of storage remaining. if sf.SizeRemaining < modules.SectorSize || sf.Size < sf.SizeRemaining { continue } winner = true // at least one storage folder has enough space for a new sector. // Check this storage folder against the current winning storage folder's utilization. sfFree := float64(sf.SizeRemaining) / float64(sf.Size) if mostFree < sfFree { mostFree = sfFree winningIndex = i } } // Do not return any storage folder if none of them have enough room for a // new sector. if !winner { return nil, -1 } return sfs[winningIndex], winningIndex } // offloadStorageFolder takes sectors in a storage folder and moves them to // another storage folder. func (sm *StorageManager) offloadStorageFolder(offloadFolder *storageFolder, dataToOffload uint64) error { // The host is going to check every sector, using a different database tx // for each sector. To be able to track progress, a starting point needs to // be grabbed. This read grabs the starting point. // // It is expected that the host is under lock for the whole operation - // this function should be the only function with access to the database. var currentSectorID []byte var currentSectorBytes []byte err := sm.db.View(func(tx *bolt.Tx) error { currentSectorID, currentSectorBytes = tx.Bucket(bucketSectorUsage).Cursor().First() return nil }) if err != nil { return err } // Create a list of available folders. As folders are filled up, this list // will be pruned. Once all folders are full, the offload loop will quit // and return with ErrIncompleteOffload. availableFolders := make([]*storageFolder, 0) for _, sf := range sm.storageFolders { if sf == offloadFolder { // The offload folder is not an available folder. continue } if sf.SizeRemaining < modules.SectorSize { // Folders that don't have enough room for a new sector are not // available. continue } availableFolders = append(availableFolders, sf) } // Go through the sectors one at a time. Sectors that are not a part of the // provided storage folder are ignored. Sectors that are a part of the // storage folder will be moved to a new storage folder. The loop will quit // after 'dataToOffload' data has been moved from the storage folder. dataOffloaded := uint64(0) for currentSectorID != nil && dataOffloaded < dataToOffload && len(availableFolders) > 0 { err = sm.db.Update(func(tx *bolt.Tx) error { // Defer seeking to the next sector. defer func() { bsuc := tx.Bucket(bucketSectorUsage).Cursor() bsuc.Seek(currentSectorID) currentSectorID, currentSectorBytes = bsuc.Next() }() // Determine whether the sector needs to be moved. var usage sectorUsage err = json.Unmarshal(currentSectorBytes, &usage) if err != nil { return err } if !bytes.Equal(usage.StorageFolder, offloadFolder.UID) { // The current sector is not in the offloading storage folder, // try the next sector. Returning nil will advance to the next // iteration of the loop. return nil } // This sector is in the removal folder, and therefore needs to // be moved to the next folder. success := false emptiestFolder, emptiestIndex := emptiestStorageFolder(availableFolders) for emptiestFolder != nil { oldSectorPath := filepath.Join(sm.persistDir, offloadFolder.uidString(), string(currentSectorID)) // Try reading the sector from disk. sectorData, err := sm.dependencies.readFile(oldSectorPath) if err != nil { // Inidicate that the storage folder is having read // troubles. offloadFolder.FailedReads++ // Returning nil will move to the next sector. Though the // current sector has failed to read, the host will keep // trying future sectors in hopes of finishing the task. return nil } // Indicate that the storage folder did a successful read. offloadFolder.SuccessfulReads++ // Try writing the sector to the emptiest storage folder. newSectorPath := filepath.Join(sm.persistDir, emptiestFolder.uidString(), string(currentSectorID)) err = sm.dependencies.writeFile(newSectorPath, sectorData, 0700) if err != nil { // Indicate that the storage folder is having write // troubles. emptiestFolder.FailedWrites++ // After the failed write, try removing any garbage that // may have gotten left behind. The error is not checked, // as it is known that the disk is having write troubles. _ = sm.dependencies.removeFile(newSectorPath) // Because the write failed, we should move on to the next // storage folder, and remove the current storage folder // from the list of available folders. availableFolders = append(availableFolders[0:emptiestIndex], availableFolders[emptiestIndex+1:]...) // Try the next folder. emptiestFolder, emptiestIndex = emptiestStorageFolder(availableFolders) continue } // Indicate that the storage folder is doing successful writes. emptiestFolder.SuccessfulWrites++ err = sm.dependencies.removeFile(oldSectorPath) if err != nil { // Indicate that the storage folder is having write // troubles. offloadFolder.FailedWrites++ } else { offloadFolder.SuccessfulWrites++ } success = true break } if !success { // The sector failed to be written successfully, try moving to // the next sector. return nil } offloadFolder.SizeRemaining += modules.SectorSize emptiestFolder.SizeRemaining -= modules.SectorSize dataOffloaded += modules.SectorSize // Update the sector usage database to reflect the file movement. // Because this cannot be done atomically, recovery tools are // required to deal with outlier cases where the swap is fine but // the database update is not. usage.StorageFolder = emptiestFolder.UID newUsageBytes, err := json.Marshal(usage) if err != nil { return err } err = tx.Bucket(bucketSectorUsage).Put(currentSectorID, newUsageBytes) if err != nil { return err } // Seek to the next sector. return nil }) if err != nil { return err } } if dataOffloaded < dataToOffload { return ErrIncompleteOffload } return nil } // storageFolder returns the storage folder in the host with the input uid. If // the storage folder is not found, nil is returned. func (sm *StorageManager) storageFolder(uid []byte) *storageFolder { for _, sf := range sm.storageFolders { if bytes.Equal(uid, sf.UID) { return sf } } return nil } // uidString returns the string value of the storage folder's UID. This string // maps to the filename of the symlink that is used to point to the folder that // holds all of the sector data contained by the storage folder. func (sf *storageFolder) uidString() string { if len(sf.UID) != storageFolderUIDSize { build.Critical("sector UID length is incorrect - perhaps the wrong version of Sia is being run?") } return hex.EncodeToString(sf.UID) } // AddStorageFolder adds a storage folder to the host. func (sm *StorageManager) AddStorageFolder(path string, size uint64) error { // Lock the host for the duration of the add operation - it is important // that the host not be manipulated while sectors are being moved around. sm.mu.Lock() defer sm.mu.Unlock() // The resource lock is required as the sector movements require access to // the logger. sm.resourceLock.RLock() defer sm.resourceLock.RUnlock() if sm.closed { return errStorageManagerClosed } // Check that the maximum number of allowed storage folders has not been // exceeded. if len(sm.storageFolders) >= maximumStorageFolders { return errMaxStorageFolders } // Check that the storage folder being added meets the size requirements. if size > maximumStorageFolderSize { return ErrLargeStorageFolder } if size < minimumStorageFolderSize { return ErrSmallStorageFolder } // Check that the path is an absolute path. if !filepath.IsAbs(path) { if path == "" { return ErrEmptyPath } return ErrRelativePath } // Check that the folder being linked to is not already in use. for _, sf := range sm.storageFolders { if sf.Path == path { return ErrRepeatFolder } } // Check that the folder being linked to both exists and is a folder. pathInfo, err := os.Stat(path) if err != nil { return err } if !pathInfo.Mode().IsDir() { return errStorageFolderNotFolder } // Create a storage folder object. newSF := &storageFolder{ Path: path, Size: size, SizeRemaining: size, } // Give the storage folder a new UID, while enforcing that the storage // folder can't have a collision with any of the other storage folders. newSF.UID = make([]byte, storageFolderUIDSize) for { // Generate an attempt UID for the storage folder. _, err = sm.dependencies.randRead(newSF.UID) if err != nil { return err } // Check for collsions. Check should be relatively inexpensive at all // times, because the total number of storage folders is limited to // 256. safe := true for _, sf := range sm.storageFolders { if bytes.Equal(newSF.UID, sf.UID) { safe = false break } } if safe { break } } // Symlink the path for the data to the UID location of the host. symPath := filepath.Join(sm.persistDir, newSF.uidString()) err = sm.dependencies.symlink(path, symPath) if err != nil { return err } // Add the storage folder to the list of folders for the host. sm.storageFolders = append(sm.storageFolders, newSF) return sm.saveSync() } // ResetStorageFolderHealth will reset the read and write statistics for the // storage folder. func (sm *StorageManager) ResetStorageFolderHealth(index int) error { sm.mu.Lock() defer sm.mu.Unlock() sm.resourceLock.RLock() defer sm.resourceLock.RUnlock() if sm.closed { return errStorageManagerClosed } // Check that the input is valid. if index >= len(sm.storageFolders) { return errBadStorageFolderIndex } // Reset the storage statistics and save the host. sm.storageFolders[index].FailedReads = 0 sm.storageFolders[index].FailedWrites = 0 sm.storageFolders[index].SuccessfulReads = 0 sm.storageFolders[index].SuccessfulWrites = 0 return sm.saveSync() } // RemoveStorageFolder removes a storage folder from the host. func (sm *StorageManager) RemoveStorageFolder(removalIndex int, force bool) error { sm.mu.Lock() defer sm.mu.Unlock() sm.resourceLock.RLock() defer sm.resourceLock.RUnlock() if sm.closed { return errStorageManagerClosed } // Check that the removal folder exists, and create a shortcut to it. if removalIndex >= len(sm.storageFolders) || removalIndex < 0 { return errBadStorageFolderIndex } removalFolder := sm.storageFolders[removalIndex] // Move all of the sectors in the storage folder to other storage folders. usedSize := removalFolder.Size - removalFolder.SizeRemaining offloadErr := sm.offloadStorageFolder(removalFolder, usedSize) // If 'force' is set, we want to ignore 'ErrIncompleteOffload' and try to // remove the storage folder anyway. For any other error, we want to halt // and return the error. if force && offloadErr == ErrIncompleteOffload { offloadErr = nil } if offloadErr != nil { return offloadErr } // Remove the storage folder from the host and then save the host. sm.storageFolders = append(sm.storageFolders[0:removalIndex], sm.storageFolders[removalIndex+1:]...) removeErr := sm.dependencies.removeFile(filepath.Join(sm.persistDir, removalFolder.uidString())) saveErr := sm.saveSync() return composeErrors(saveErr, removeErr) } // ResizeStorageFolder changes the amount of disk space that is going to be // allocated to a storage folder. func (sm *StorageManager) ResizeStorageFolder(storageFolderIndex int, newSize uint64) error { // Lock the host for the duration of the resize operation - it is important // that the host not be manipulated while sectors are being moved around. sm.mu.Lock() defer sm.mu.Unlock() // The resource lock is required as the sector movements require access to // the logger. sm.resourceLock.RLock() defer sm.resourceLock.RUnlock() if sm.closed { return errStorageManagerClosed } // Check that the inputs are valid. if storageFolderIndex >= len(sm.storageFolders) || storageFolderIndex < 0 { return errBadStorageFolderIndex } resizeFolder := sm.storageFolders[storageFolderIndex] if newSize > maximumStorageFolderSize { return ErrLargeStorageFolder } if newSize < minimumStorageFolderSize { return ErrSmallStorageFolder } if resizeFolder.Size == newSize { return ErrNoResize } // Sectors do not need to be moved onto or away from the resize folder if // the folder is growing, or if after being shrunk the folder still has // enough storage to house all of the sectors it currently tracks. resizeFolderSizeConsumed := resizeFolder.Size - resizeFolder.SizeRemaining if resizeFolderSizeConsumed <= newSize { resizeFolder.SizeRemaining = newSize - resizeFolderSizeConsumed resizeFolder.Size = newSize return sm.saveSync() } // Calculate the number of sectors that need to be offloaded from the // storage folder. offloadSize := resizeFolderSizeConsumed - newSize offloadErr := sm.offloadStorageFolder(resizeFolder, offloadSize) if offloadErr == ErrIncompleteOffload { // Offloading has not fully succeeded, but may have partially // succeeded. To prevent new sectors from being added to the storage // folder, clamp the size of the storage folder to the current amount // of storage in use. resizeFolder.Size -= resizeFolder.SizeRemaining resizeFolder.SizeRemaining = 0 return offloadErr } else if offloadErr != nil { return offloadErr } resizeFolder.Size = newSize resizeFolder.SizeRemaining = 0 return sm.saveSync() } // StorageFolders provides information about all of the storage folders in the // host. func (sm *StorageManager) StorageFolders() (sfms []modules.StorageFolderMetadata) { sm.mu.RLock() defer sm.mu.RUnlock() for _, sf := range sm.storageFolders { sfms = append(sfms, modules.StorageFolderMetadata{ Capacity: sf.Size, CapacityRemaining: sf.SizeRemaining, Path: sf.Path, FailedReads: sf.FailedReads, FailedWrites: sf.FailedWrites, SuccessfulReads: sf.SuccessfulReads, SuccessfulWrites: sf.SuccessfulWrites, }) } return sfms } Sia-1.0.4/modules/host/storagemanager/storagefolders_errors_test.go000066400000000000000000000054071302033070500256600ustar00rootroot00000000000000package storagemanager import ( "errors" "path/filepath" "testing" "github.com/NebulousLabs/Sia/modules" ) // TestClosedStorageManagerOperations tries a bunch of operations on the storage manager // after it has been closed. func TestClosedStorageManagerOperations(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() smt, err := newStorageManagerTester("TestClosedStorageManagerOperations") if err != nil { t.Fatal(err) } // Close the storage manager. err = smt.sm.Close() if err != nil { t.Fatal(err) } defer smt.Close() err = smt.sm.AddStorageFolder(filepath.Join(smt.persistDir, modules.StorageManagerDir), minimumStorageFolderSize) if err != errStorageManagerClosed { t.Fatal("expected errStorageManagerClosed:", err) } err = smt.sm.RemoveStorageFolder(1, true) if err != errStorageManagerClosed { t.Fatal("expected errStorageManagerClosed:", err) } err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize) if err != errStorageManagerClosed { t.Fatal("expected errStorageManagerClosed:", err) } // Number of storage folders should still be zero. if len(smt.sm.storageFolders) != 0 { t.Error("storage folder should not have been added to the storage manager as the storage manager is closed.") } } // faultyRand is a mocked filesystem which can be configured to fail for certain // files and folders. type faultyRand struct { productionDependencies } // errMockBadRand is returned when a mocked dependency is intentionally // returning an error instead of randomly generating data. var errMockBadRand = errors.New("mocked randomness is intentionally failing") // randRead replaces the production dependency crypto/rand.Read with a faulty // reader - an error is always returned. func (faultyRand) randRead([]byte) (int, error) { return 0, errMockBadRand } // TestAddFolderNoRand tries adding a folder to the storage manager when the // cryptographic randomness generator is not working. func TestAddFolderNoRand(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() smt, err := newStorageManagerTester("TestAddFolderNoRand") if err != nil { t.Fatal(err) } defer smt.Close() // Replace the storage manager with one that cannot do randomness // operations. err = smt.sm.Close() if err != nil { t.Fatal(err) } smt.sm, err = newStorageManager(faultyRand{}, filepath.Join(smt.persistDir, modules.StorageManagerDir)) if err != nil { t.Fatal(err) } err = smt.sm.AddStorageFolder(filepath.Join(smt.persistDir, modules.StorageManagerDir), minimumStorageFolderSize) if err != errMockBadRand { t.Fatal(err) } // Number of storage folders should be zero. storageFolders := smt.sm.StorageFolders() if len(storageFolders) != 0 { t.Error("storage folder was added to the storage manager despite a dependency failure") } } Sia-1.0.4/modules/host/storagemanager/storagefolders_smoke_errors_test.go000066400000000000000000000371311302033070500270550ustar00rootroot00000000000000package storagemanager import ( "bytes" "io/ioutil" "os" "path/filepath" "strings" "testing" "github.com/NebulousLabs/Sia/modules" ) // faultyFS is a mocked filesystem which can be configured to fail for certain // files and folders, as indicated by 'brokenSubstrings'. type faultyFS struct { // brokenSubstrings is a list of substrings that, when appearing in a // filepath, will cause the call to fail. brokenSubstrings []string productionDependencies } // readFile reads a file from the filesystem. The call will fail if reading // from a file that has a substring which matches the ffs list of broken // substrings. func (ffs faultyFS) readFile(s string) ([]byte, error) { for _, bs := range ffs.brokenSubstrings { if strings.Contains(s, bs) { return nil, mockErrReadFile } } return ffs.productionDependencies.readFile(s) } // symlink creates a symlink between a source and a destination file, but will // fail if either filename contains a substring found in the set of broken // substrings. func (ffs faultyFS) symlink(s1, s2 string) error { for _, bs := range ffs.brokenSubstrings { if strings.Contains(s1, bs) || strings.Contains(s2, bs) { return mockErrSymlink } } return ffs.productionDependencies.symlink(s1, s2) } // writeFile reads a file from the filesystem. The call will fail if reading // from a file that has a substring which matches the ffs list of broken // substrings. func (ffs faultyFS) writeFile(s string, b []byte, fm os.FileMode) error { // The partial write reqires that there be at least a few bytes, so that a // partial write can be properly simulated. if len(b) < 2 { panic("mocked writeFile requires file data that's at least 2 bytes in length") } for _, bs := range ffs.brokenSubstrings { if strings.Contains(s, bs) { // Do a partial write, so that garbase is left on the filesystem // that the code should be trying to clean up. err := ioutil.WriteFile(s, b[:len(b)/2], fm) if err != nil { return err } // Return a simulated failure, as the full slice was not written. return mockErrWriteFile } } return ioutil.WriteFile(s, b, fm) } // faultyRemove is a mocked set of dependencies that operates as normal except // that removeFile will fail. type faultyRemove struct { productionDependencies } // removeFile fails to remove a file from the filesystem. func (faultyRemove) removeFile(s string) error { return mockErrRemoveFile } // TestStorageFolderTolerance tests the tolerance of storage folders in the // presence of disk failures. Disk failures should be recorded, and the // failures should be handled gracefully - nonfailing disks should not have // problems. func TestStorageFolderTolerance(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() smt, err := newStorageManagerTester("TestStorageFolderTolerance") if err != nil { t.Fatal(err) } defer smt.Close() // Replace the storage manager so that it's using faultyOS for its // dependencies. err = smt.sm.Close() if err != nil { t.Fatal(err) } ffs := new(faultyFS) smt.sm, err = newStorageManager(ffs, filepath.Join(smt.persistDir, modules.StorageManagerDir)) if err != nil { t.Fatal(err) } // Add a storage folder when the symlinking is failing. storageFolderOne := filepath.Join(smt.persistDir, "driveOne") ffs.brokenSubstrings = []string{storageFolderOne} err = os.Mkdir(storageFolderOne, 0700) if err != nil { t.Fatal(err) } err = smt.sm.AddStorageFolder(storageFolderOne, minimumStorageFolderSize) if err != mockErrSymlink { t.Fatal(err) } // Add storage folder one without errors, and then add a sector to the // storage folder. ffs.brokenSubstrings = nil err = smt.sm.AddStorageFolder(storageFolderOne, minimumStorageFolderSize) if err != nil { t.Fatal(err) } sectorRoot, sectorData, err := createSector() if err != nil { t.Fatal(err) } err = smt.sm.AddSector(sectorRoot, 10, sectorData) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the storage manager, to verify that the // persistence structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Check the filesystem - there should be one sector in the storage folder. infos, err := ioutil.ReadDir(storageFolderOne) if err != nil { t.Fatal(err) } if len(infos) != 1 { t.Fatal("expecting at least one sector in storage folder one") } // Replace the storage manager dependencies with the faulty remove, and // then try to remove the sector. smt.sm.dependencies = faultyRemove{} err = smt.sm.RemoveSector(sectorRoot, 10) if err != mockErrRemoveFile { t.Fatal(err) } // Check that the failed write count was incremented for the storage // folder. if smt.sm.storageFolders[0].FailedWrites != 1 { t.Fatal("failed writes counter is not incrementing properly") } // Check the filesystem - sector should still be in the storage folder. infos, err = ioutil.ReadDir(storageFolderOne) if err != nil { t.Fatal(err) } if len(infos) != 1 { t.Fatal("expecting at least one sector in storage folder one") } // Put 'ffs' back as the set of dependencies. smt.sm.dependencies = ffs // Add a second storage folder, which can receive the sector when the first // storage folder is deleted. storageFolderTwo := filepath.Join(smt.persistDir, "driveTwo") err = os.Mkdir(storageFolderTwo, 0700) if err != nil { t.Fatal(err) } err = smt.sm.AddStorageFolder(storageFolderTwo, minimumStorageFolderSize*3) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the storage manager, to verify that the // persistence structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Trigger read errors in storage folder one, which means the storage // folder is not going to be able to be deleted successfully. ffs.brokenSubstrings = []string{filepath.Join(smt.persistDir, modules.StorageManagerDir, smt.sm.storageFolders[0].uidString())} err = smt.sm.RemoveStorageFolder(0, false) if err != ErrIncompleteOffload { t.Fatal(err) } // Check that the storage folder was not removed. if len(smt.sm.storageFolders) != 2 { t.Fatal("expecting two storage folders after failed remove") } // Check that the read failure was documented. if smt.sm.storageFolders[0].FailedReads != 1 { t.Error("expecting a read failure to be reported:", smt.sm.storageFolders[0].FailedReads) } // Check the filesystem - there should be one sector in the storage folder, // and none in storage folder two. infos, err = ioutil.ReadDir(storageFolderOne) if err != nil { t.Fatal(err) } if len(infos) != 1 { t.Fatal("expecting at least one sector in storage folder one") } infos, err = ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } if len(infos) != 0 { t.Fatal("expecting zero sectors in storage folder two") } // Switch the failure from a read error in the source folder to a write // error in the destination folder. ffs.brokenSubstrings = []string{filepath.Join(smt.persistDir, modules.StorageManagerDir, smt.sm.storageFolders[1].uidString())} err = smt.sm.RemoveStorageFolder(0, false) if err != ErrIncompleteOffload { t.Fatal(err) } // Do a probabilistic reset of the storage manager, to verify that the // persistence structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Check that the storage folder was not removed. if len(smt.sm.storageFolders) != 2 { t.Fatal("expecting two storage folders after failed remove") } // Check that the read failure was documented. if smt.sm.storageFolders[1].FailedWrites != 1 { t.Error("expecting a read failure to be reported:", smt.sm.storageFolders[1].FailedWrites) } // Check the filesystem - there should be one sector in the storage folder, // and none in storage folder two. infos, err = ioutil.ReadDir(storageFolderOne) if err != nil { t.Fatal(err) } if len(infos) != 1 { t.Fatal("expecting at least one sector in storage folder one") } infos, err = ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } if len(infos) != 0 { t.Fatal("expecting zero sectors in storage folder two") } // Try to forcibly remove the first storage folder, while in the presence // of read errors. ffs.brokenSubstrings = []string{filepath.Join(smt.persistDir, modules.StorageManagerDir, smt.sm.storageFolders[0].uidString())} uid2 := smt.sm.storageFolders[1].UID err = smt.sm.RemoveStorageFolder(0, true) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the storage manager, to verify that the // persistence structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Check that the storage folder was removed. if len(smt.sm.storageFolders) != 1 { t.Fatal("expecting two storage folders after failed remove") } if !bytes.Equal(uid2, smt.sm.storageFolders[0].UID) { t.Fatal("storage folder was not removed correctly") } // Check the filesystem - there should be no sectors in storage folder two. infos, err = ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } if len(infos) != 0 { t.Fatal("expecting zero sectors in storage folder two") } // Add a storage folder with room for sectors. Because storageFolderOne has // leftover sectors that the program was unable to clean up (due to disk // failure), a third storage folder will be created. ffs.brokenSubstrings = nil storageFolderThree := filepath.Join(smt.persistDir, "driveThree") err = os.Mkdir(storageFolderThree, 0700) if err != nil { t.Fatal(err) } err = smt.sm.AddStorageFolder(storageFolderThree, minimumStorageFolderSize+modules.SectorSize) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the storage manager, to verify that the // persistence structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Fill up the second storage folder, so that resizes can be attempted with // failing disks. storageFolderOne has enough space to store the sectors, // but is having disk troubles. ffs.brokenSubstrings = []string{filepath.Join(smt.persistDir, modules.StorageManagerDir, smt.sm.storageFolders[1].uidString())} numSectors := (minimumStorageFolderSize * 3) / modules.SectorSize for i := uint64(0); i < numSectors; i++ { sectorRoot, sectorData, err := createSector() if err != nil { t.Fatal(err) } err = smt.sm.AddSector(sectorRoot, 11, sectorData) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the storage manager, to verify that the // persistence structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } } // Check the filesystem - storage folder one is having disk issues and // should have no sectors. Storage folder two should be full. infos, err = ioutil.ReadDir(storageFolderThree) if err != nil { t.Fatal(err) } if len(infos) != 0 { t.Fatal("expecting zero sectors in storage folder one") } infos, err = ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } if len(infos) != int(numSectors) { t.Fatal("expecting", numSectors, "sectors in storage folder two") } // Try adding another sector, there should be an error because the one disk // is full and the other is having disk troubles. sectorRoot, sectorData, err = createSector() if err != nil { t.Fatal(err) } err = smt.sm.AddSector(sectorRoot, 11, sectorData) if err != errDiskTrouble { t.Fatal(err) } // Do a probabilistic reset of the storage manager, to verify that the // persistence structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Check the filesystem - storage folder one is having disk issues and // should have no sectors. Storage folder two should be full. infos, err = ioutil.ReadDir(storageFolderThree) if err != nil { t.Fatal(err) } if len(infos) != 0 { t.Fatal("expecting zero sectors in storage folder one") } infos, err = ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } if len(infos) != int(numSectors) { t.Fatal("expecting", numSectors, "sectors in storage folder two") } // Add a third storage folder. Then try to resize the second storage folder // such that both storageFolderThree and storageFolderFour have room for // the data, but only storageFolderFour is not haivng disk troubles. storageFolderFour := filepath.Join(smt.persistDir, "driveFour") err = os.Mkdir(storageFolderFour, 0700) if err != nil { t.Fatal(err) } err = smt.sm.AddStorageFolder(storageFolderFour, minimumStorageFolderSize) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the storage manager, to verify that the // persistence structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize*2) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the storage manager, to verify that the // persistence structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Check the filesystem - storageFolderTwo should have // minimumStorageFolderSize*2 worth of sectors, and storageFolderFour // should have minimumStorageFolderSize worth of sectors. infos, err = ioutil.ReadDir(storageFolderThree) if err != nil { t.Fatal(err) } if len(infos) != 0 { t.Fatal("expecting zero sectors in storage folder three") } infos, err = ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } if len(infos) != int(numSectors)-int(minimumStorageFolderSize/modules.SectorSize) { t.Fatal("expecting", numSectors, "sectors in storage folder two") } infos, err = ioutil.ReadDir(storageFolderFour) if err != nil { t.Fatal(err) } if len(infos) != int(minimumStorageFolderSize/modules.SectorSize) { t.Fatal("expecting to have 8 sectors in storageFolderFour") } // Trigger an incomplete disk transfer by adding room for one more sector // to storageFolderFour, but then trying to remove a bunch of sectors from // storageFolderTwo. There is enough room on storage folder 3 to make the // operation successful, but it is having disk troubles. err = smt.sm.ResizeStorageFolder(2, minimumStorageFolderSize+modules.SectorSize) if err != nil { t.Fatal(err) } err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize) if err != ErrIncompleteOffload { t.Fatal(err) } // Do a probabilistic reset of the storage manager, to verify that the // persistence structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Check that the sizes of the storage folders have been updated correctly. if smt.sm.storageFolders[0].Size != minimumStorageFolderSize*2-modules.SectorSize { t.Error("storage folder size was not decreased correctly during the shrink operation") } if smt.sm.storageFolders[0].SizeRemaining != 0 { t.Error("storage folder size remaining was not updated correctly after failed shrink operation") } // Check the filesystem - there should be one less sector in // storageFolderTwo from the previous check, and one more sector in // storageFolderFour. infos, err = ioutil.ReadDir(storageFolderThree) if err != nil { t.Fatal(err) } if len(infos) != 0 { t.Fatal("expecting zero sectors in storage folder three") } infos, err = ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } if len(infos) != int(numSectors)-int(minimumStorageFolderSize/modules.SectorSize)-1 { t.Fatal("expecting", numSectors, "sectors in storage folder two") } infos, err = ioutil.ReadDir(storageFolderFour) if err != nil { t.Fatal(err) } if len(infos) != int(minimumStorageFolderSize/modules.SectorSize)+1 { t.Fatal("filesystem consistency error") } } Sia-1.0.4/modules/host/storagemanager/storagefolders_smoke_test.go000066400000000000000000000757051302033070500254720ustar00rootroot00000000000000package storagemanager import ( "bytes" "encoding/json" "errors" "io/ioutil" "os" "path/filepath" "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // sectorUsageCheck compares a manually maintained sector usage map to the // manager's internal sector usage map, and returns an error if there are any // inconsistencies. func (smt *storageManagerTester) sectorUsageCheck(sectorUsageMap map[crypto.Hash][]types.BlockHeight) error { // Check that the in-database representation for the sector usage map // matches the in-memory understanding of what the sector map should be return smt.sm.db.View(func(tx *bolt.Tx) error { bsu := tx.Bucket(bucketSectorUsage) // Make sure that the number of sectors in the sector usage map and the // number of sectors in the database are the same. if len(sectorUsageMap) != bsu.Stats().KeyN { return errors.New("BucketSectorUsage has the wrong number of sectors recorded") } // For every sector in the sector usage map, make sure the database has // a matching sector with the right expiry information. for sectorRoot, expiryHeights := range sectorUsageMap { usageBytes := bsu.Get(smt.sm.sectorID(sectorRoot[:])) if usageBytes == nil { return errors.New("no usage info on known sector") } var usage sectorUsage err := json.Unmarshal(usageBytes, &usage) if err != nil { return err } if len(usage.Expiry) != len(expiryHeights) { return errors.New("usage information mismatch") } for i, expiryHeight := range usage.Expiry { if expiryHeight != expiryHeights[i] { // The correctness could be made not-implementation // dependent by sorting the two arrays before comparing // them, but that was deemed an unneeded step for this // test. return errors.New("usage expiry height mismatch - correctness is implementation dependent") } } } return nil }) } // TestStorageFolderUsage is a general integration test which tries all of the // major storage folder operations in various orders, all while adding and // removing sectors to verify that the behavior works as expected. func TestStorageFolderUsage(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() smt, err := newStorageManagerTester("TestStorageFolderUsage") if err != nil { t.Fatal(err) } // Start by checking that the initial state of the manager has no storage // added to it. totalStorage, remainingStorage := smt.sm.capacity() if totalStorage != 0 || remainingStorage != 0 { t.Error("initial capacity of manager is not reported at 0 - but no drives have been added!") } // Try adding a sector when there are no storage folders. sectorRoot, sectorData, err := createSector() if err != nil { t.Fatal(err) } err = smt.sm.AddSector(sectorRoot, 10, sectorData) if err != errInsufficientStorageForSector { t.Fatal(err) } // Add a storage folder, simulating a new drive being connected to the // manager. storageFolderOne := filepath.Join(smt.persistDir, "manager drive 1") // Try using a file size that is too small. Because a filesize check is // quicker than a disk check, the filesize check should come first. err = smt.sm.AddStorageFolder(storageFolderOne, minimumStorageFolderSize-1) if err != ErrSmallStorageFolder { t.Fatal("expecting ErrSmallStorageFolder:", err) } // Try a file size that is too large. err = smt.sm.AddStorageFolder(storageFolderOne, maximumStorageFolderSize+1) if err != ErrLargeStorageFolder { t.Fatal("expecting ErrLargeStorageFolder:", err) } // Try linking to a storage folder that does not exist. err = smt.sm.AddStorageFolder(storageFolderOne, minimumStorageFolderSize) if err == nil { t.Fatal("should not be able to link to a storage folder which does not exist") } // Try linking to a storage folder that's not a directory. err = ioutil.WriteFile(storageFolderOne, make([]byte, minimumStorageFolderSize), 0700) if err != nil { t.Fatal(err) } err = smt.sm.AddStorageFolder(storageFolderOne, minimumStorageFolderSize) if err != errStorageFolderNotFolder { t.Fatal(err) } // Try linking to a storage folder that is a directory. err = os.Remove(storageFolderOne) if err != nil { t.Fatal(err) } err = os.Mkdir(storageFolderOne, 0700) if err != nil { t.Fatal(err) } err = smt.sm.AddStorageFolder(storageFolderOne, minimumStorageFolderSize) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the manager, to verify that the persistence // structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Check that the manager has correctly updated the amount of total storage. totalStorage, remainingStorage = smt.sm.capacity() if totalStorage != minimumStorageFolderSize || remainingStorage != minimumStorageFolderSize { t.Error("manager capacity has not been correctly updated after adding a storage folder") t.Error(totalStorage, minimumStorageFolderSize, remainingStorage) } // Add a second storage folder. storageFolderTwo := filepath.Join(smt.persistDir, "managerDrive2") err = os.Mkdir(storageFolderTwo, 0700) if err != nil { t.Fatal(err) } err = smt.sm.AddStorageFolder(storageFolderTwo, minimumStorageFolderSize*2) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the manager, to verify that the persistence // structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Check that the manager has correctly updated the amount of total // storage. totalStorage, remainingStorage = smt.sm.capacity() if totalStorage != minimumStorageFolderSize*3 || remainingStorage != minimumStorageFolderSize*3 { t.Error("manager capacity has not been correctly updated after adding a storage folder") } // Try removing the storage folder using illegal values. err = smt.sm.RemoveStorageFolder(-1, false) if err != errBadStorageFolderIndex { t.Fatal(err) } err = smt.sm.RemoveStorageFolder(2, false) if err != errBadStorageFolderIndex { t.Fatal(err) } // Try removing the second storage folder. Before removing the storage // folder, grab the path of the symlink so we can check later that it was // properly removed from the filesystem. symPath := filepath.Join(smt.sm.persistDir, smt.sm.storageFolders[1].uidString()) // Remove the storage folder. err = smt.sm.RemoveStorageFolder(1, false) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the manager, to verify that the persistence // structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Check that the manager has correctly updated the amount of total // storage. totalStorage, remainingStorage = smt.sm.capacity() if totalStorage != minimumStorageFolderSize || remainingStorage != minimumStorageFolderSize { t.Error("manager capacity has not been correctly updated after adding a storage folder") } _, err = os.Stat(symPath) if err == nil || !os.IsNotExist(err) { t.Error("Does not appear that the sympath was removed from disk:", err) } // No sectors added yet, the storage folder statistics should all be clean. for _, sf := range smt.sm.storageFolders { if sf.SuccessfulReads != 0 || sf.SuccessfulWrites != 0 || sf.FailedReads != 0 || sf.FailedWrites != 0 { t.Error("storage folder does not have blank health stats") } } // Retry adding the sector, the add should succeed and the amount of // remaining storage should be updated. sectorExpiry := types.BlockHeight(10) err = smt.sm.AddSector(sectorRoot, sectorExpiry, sectorData) if err != nil { t.Fatal(err) } // Check that the capacity has updated to reflected the new sector. totalStorage, remainingStorage = smt.sm.capacity() if totalStorage != minimumStorageFolderSize || remainingStorage != minimumStorageFolderSize-modules.SectorSize { t.Error("manager capacity has not been correctly updated after adding a sector", totalStorage, remainingStorage) } // Check that the sector has been added to the filesystem correctly - the // file should exist in storageFolderOne, and the data in the file should // match the data of the sector. sectorPath := filepath.Join(storageFolderOne, string(smt.sm.sectorID(sectorRoot[:]))) err = func() error { sectorFile, err := os.Open(sectorPath) defer sectorFile.Close() fileInfo, err := sectorFile.Stat() if err != nil { return err } if uint64(fileInfo.Size()) != modules.SectorSize { return errors.New("scanned sector is not the right size") } readSectorData, err := ioutil.ReadAll(sectorFile) if err != nil { return err } if bytes.Compare(readSectorData, sectorData) != 0 { return errors.New("read sector does not match sector data") } return nil }() if err != nil { t.Fatal(err) } // Check that the sector as represented in the database has the correct // height values. err = smt.sm.db.View(func(tx *bolt.Tx) error { bsu := tx.Bucket(bucketSectorUsage) usageBytes := bsu.Get(smt.sm.sectorID(sectorRoot[:])) var usage sectorUsage err := json.Unmarshal(usageBytes, &usage) if err != nil { return err } if len(usage.Expiry) != 1 { return errors.New("wrong usage expiry length in BucketSectorUsage") } if usage.Expiry[0] != 10 { return errors.New("usage expiry for sector is set to the wrong height") } return nil }) if err != nil { t.Fatal(err) } // Check that the disk health stats match the expected values. for _, sf := range smt.sm.storageFolders { if sf.SuccessfulReads != 0 || sf.SuccessfulWrites != 1 || sf.FailedReads != 0 || sf.FailedWrites != 0 { t.Error("storage folder does not have blank health stats") } } // Try to resize the storage folder. While resizing the storage folder, try // a bunch of invalid resize calls. err = smt.sm.ResizeStorageFolder(1, minimumStorageFolderSize-1) if err != errBadStorageFolderIndex { t.Error(err) } err = smt.sm.ResizeStorageFolder(-1, minimumStorageFolderSize-1) if err != errBadStorageFolderIndex { t.Error(err) } err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize-1) if err != ErrSmallStorageFolder { t.Error(err) } err = smt.sm.ResizeStorageFolder(0, maximumStorageFolderSize+1) if err != ErrLargeStorageFolder { t.Error(err) } err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize*10) if err != nil { t.Fatal(err) } err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize*10) if err != ErrNoResize { t.Fatal(err) } // Do a probabilistic reset of the manager, to verify that the persistence // structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Manager should be able to support having uneven storage sizes. oddStorageSize := (minimumStorageFolderSize) + modules.SectorSize*3 + 3 err = smt.sm.ResizeStorageFolder(0, oddStorageSize) if err != nil { t.Fatal(err) } // Create a sector list, containing all sectors (including repeats) and the // heights at which they expire. This sector list will be updated as // sectors are added and removed. sectorUsageMap := make(map[crypto.Hash][]types.BlockHeight) sectorUsageMap[sectorRoot] = []types.BlockHeight{sectorExpiry} // Sanity check - manager should not have any sectors in it. totalStorage, remainingStorage = smt.sm.capacity() if totalStorage != remainingStorage+modules.SectorSize { t.Fatal("manager is not empty at the moment of creating the in-memory sector usage map") } // Verify that the initial sector usage map was created correctly. err = smt.sectorUsageCheck(sectorUsageMap) if err != nil { t.Fatal(err) } // Fill the storage folder above the minimum size, then try to shrink it to // the minimum size. for i := uint64(0); i <= minimumStorageFolderSize/modules.SectorSize; i++ { sectorRoot, sectorData, err := createSector() if err != nil { t.Fatal(err) } err = smt.sm.AddSector(sectorRoot, 86+types.BlockHeight(i), sectorData) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the manager, to verify that the persistence // structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Now that there is a sector usage map, it must be kept consistent // with the sector usage in the manager. sectorUsageMap[sectorRoot] = []types.BlockHeight{86 + types.BlockHeight(i)} } oldSize := smt.sm.storageFolders[0].Size err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize) if err != ErrIncompleteOffload { t.Fatal(err) } size := smt.sm.storageFolders[0].Size sizeRemaining := smt.sm.storageFolders[0].SizeRemaining if size >= oldSize || sizeRemaining > 0 { t.Fatal("manager did not correctly update the size remaining after an incomplete shrink") } // Try adding another sector, there should not be enough room. sr, sd, err := createSector() if err != nil { t.Fatal(err) } err = smt.sm.AddSector(sr, 186, sd) if err != errInsufficientStorageForSector { t.Fatal(err) } // Add a second folder, and add a sector to that folder. There should be // enough space remaining in the first folder for the removal to be // successful. err = smt.sm.AddStorageFolder(storageFolderTwo, minimumStorageFolderSize*2) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the manager, to verify that the persistence // structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } sectorRoot, sectorData, err = createSector() if err != nil { t.Fatal(err) } err = smt.sm.AddSector(sectorRoot, 81, sectorData) if err != nil { t.Fatal(err) } sectorUsageMap[sectorRoot] = []types.BlockHeight{81} // Check that the sector ended up in the right storage folder - because the // second storage folder is the least full, the sector should end up there. folderTwoUsage := smt.sm.storageFolders[1].Size - smt.sm.storageFolders[1].SizeRemaining if folderTwoUsage != modules.SectorSize { t.Error("sector did not appear to land in the right storage folder") } // Check the filesystem. The folder for storage folder 1 should have 10 // files, and the folder for storage folder 2 should have 1 file. infos, err := ioutil.ReadDir(storageFolderOne) if err != nil { t.Fatal(err) } if len(infos) != 10 { t.Fatal("storage folder one should have 10 sectors in it") } infos, err = ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } if len(infos) != 1 { t.Fatal("storage folder two should have 1 sector in it") } // The first storage folder has more sectors than the minimum allowed // amount. Reduce the size of the first storage folder to minimum, which // should be accepted but will result in sectors being transferred to the // second storage folder. totalStorage, remainingStorage = smt.sm.capacity() prevStorage := totalStorage usedStorage := totalStorage - remainingStorage err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize) if err != nil { t.Fatal(err) } totalStorage, remainingStorage = smt.sm.capacity() if usedStorage != totalStorage-remainingStorage { t.Error("the used storage value adjusted after removing a storage folder", usedStorage, totalStorage-remainingStorage) } if totalStorage >= prevStorage { t.Error("total storage was not adjusted correctly after removing a storage folder") } // Check the filesystem. infos, err = ioutil.ReadDir(storageFolderOne) if err != nil { t.Fatal(err) } if len(infos) != 8 { t.Fatal("wrong number of sectors in storage folder one") } infos, err = ioutil.ReadDir(storageFolderTwo) if len(infos) != 3 { t.Fatal("wrong number of sectors in storage folder two") } // Remove the first storage folder, which should result in all of the // sectors being moved to the second storage folder. Note that // storageFolderTwo now has an index of '0'. totalStorage, remainingStorage = smt.sm.capacity() prevStorage = totalStorage usedStorage = totalStorage - remainingStorage symPath = filepath.Join(smt.sm.persistDir, smt.sm.storageFolders[0].uidString()) err = smt.sm.RemoveStorageFolder(0, false) if err != nil { t.Fatal(err) } totalStorage, remainingStorage = smt.sm.capacity() if usedStorage != totalStorage-remainingStorage { t.Error("the used storage value adjusted after removing a storage folder", usedStorage, totalStorage-remainingStorage) } if totalStorage == prevStorage { t.Error("total storage was not adjusted after removing a storage folder") } // Check that the filesystem seems correct. infos, err = ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } if len(infos) != 11 { t.Fatal("wrong number of sectors in folder") } _, err = os.Stat(symPath) if !os.IsNotExist(err) { t.Fatal("the sym link to the deleted storage folder should no longer exist") } // Add the first storage folder, resize the second storage folder back down // to minimum. Note that storageFolderOne now has an index of '1', and // storageFolderTwo now has an index of '0'. err = smt.sm.AddStorageFolder(storageFolderOne, minimumStorageFolderSize) if err != nil { t.Fatal(err) } err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the manager, to verify that the persistence // structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Check the filesystem. infos, err = ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } if len(infos) != 8 { t.Fatal("wrong number of sectors") } infos, err = ioutil.ReadDir(storageFolderOne) if err != nil { t.Fatal(err) } if len(infos) != 3 { t.Fatal("wrong number of sectors") } // Add a bunch of sectors and repeat sectors at multiple colliding heights. // Start by resizing the first storage folder so that there is enough room // for the new sectors. err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize*3) if err != nil { t.Fatal(err) } for i := types.BlockHeight(0); i < 10; i++ { // Add 10 unique sectors to the map. sectorRoot, sectorData, err := createSector() if err != nil { t.Fatal(err) } for j := types.BlockHeight(0); j < 5; j++ { // Add the unique sectors at multiple heights, creating virtual // sectors. for k := types.BlockHeight(0); k < 4; k++ { // Add in an extra loop so that height collisions can be // created such that the collisions happen out of order. // Sectors are added at height 10+j+k, which means that there // will be many collisions for each height, but the collisions // are not happening in sorted order. The manager is not // expected to do sorting, but should also not be confused by a // random order. err = smt.sm.AddSector(sectorRoot, 10+j+k, sectorData) if err != nil { t.Fatal(err) } // Add the sector to the sectorUsageMap, so it can be deleted // later. expiryList, exists := sectorUsageMap[sectorRoot] if exists { sectorUsageMap[sectorRoot] = append(expiryList, 10+j+k) } else { sectorUsageMap[sectorRoot] = []types.BlockHeight{10 + j + k} } } // Do a probabilistic reset of the manager, to verify that the // persistence structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } } } // Check that the amount of storage in use represents 10 sectors, and not // more - all the others are repeats and shouldn't be counted. totalStorage, remainingStorage = smt.sm.capacity() if totalStorage != minimumStorageFolderSize*4 || remainingStorage != minimumStorageFolderSize*4-modules.SectorSize*21 { t.Fatal("Manager not reporting expected storage capacity:", totalStorage, remainingStorage, minimumStorageFolderSize*4, minimumStorageFolderSize*4-modules.SectorSize*21) } // Check that the internal sector usage database of the manager has been // updated correctly. err = smt.sectorUsageCheck(sectorUsageMap) if err != nil { t.Fatal(err) } // Check the filesystem. infos, err = ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } if len(infos) != 16 { t.Fatal("there should be 16 sectors in storage folder two") } infos, err = ioutil.ReadDir(storageFolderOne) if err != nil { t.Fatal(err) } if len(infos) != 5 { t.Fatal("there should be 5 sectors in storage folder one") } // Try removing a non-repeat sector. expiryHeights, exists := sectorUsageMap[sectorRoot] if !exists || len(expiryHeights) != 1 { t.Fatal("sector map doesn't match testing assumptions") } // Try some illegal sector removal operations before trying a legal one. err = smt.sm.RemoveSector(sectorRoot, sectorExpiry+50e6) if err != ErrSectorNotFound { t.Fatal("wrong error when removing illegal sector:", err) } alteredRoot := sectorRoot alteredRoot[0]++ err = smt.sm.RemoveSector(alteredRoot, 81) if err != ErrSectorNotFound { t.Fatal("wrong error when removing illegal sector:", err) } // Now try the legal sector removal. sectorPath = filepath.Join(storageFolderOne, string(smt.sm.sectorID(sectorRoot[:]))) err = smt.sm.RemoveSector(sectorRoot, 81) if err != nil { t.Fatal(err) } // Do a probabilistic reset of the manager, to verify that the persistence // structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Update the sector usage map to reflect the departure of a sector. delete(sectorUsageMap, sectorRoot) // Check that the new capacity is being reported correctly. totalStorage, remainingStorage = smt.sm.capacity() if totalStorage != minimumStorageFolderSize*4 || remainingStorage != minimumStorageFolderSize*4-modules.SectorSize*20 { t.Fatal("Manager not reporting expected storage capacity:") } // Run a sector usage check to make sure the manager is properly handling // the usage information when deleting a sector. err = smt.sectorUsageCheck(sectorUsageMap) if err != nil { t.Fatal(err) } // Check that the sector on-disk has been deleted. _, err = os.Stat(sectorPath) if !os.IsNotExist(err) { t.Fatal(err) } // Check that the total number of sectors seen on disk is 20. infos, err = ioutil.ReadDir(storageFolderOne) if err != nil { t.Fatal(err) } infos2, err := ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } if len(infos)+len(infos2) != 20 { t.Fatal("there should be 20 sectors total on disk at this point") } // Remove two of the duplicated sectors, one copy at a time, to see that // the database is still updating correctly. var secondIteration bool var targetedRoots []crypto.Hash for sectorRoot, expiryHeights := range sectorUsageMap { if len(expiryHeights) < 2 { continue } targetedRoots = append(targetedRoots, sectorRoot) // Break on the second iteration. if secondIteration { break } secondIteration = true } // Remove, one piece at a time, the two targeted sectors. for i, root := range targetedRoots { // Grab the initial remaining storage, to make sure that it's not being // changed when one instance of a repeated sector is removed. _, initialRemainingStorage := smt.sm.capacity() // Remove the heights one at a time. expiryHeights := sectorUsageMap[root] for len(expiryHeights) > 0 { // Check that the remaining storage is still the same. _, remainingStorage := smt.sm.capacity() if remainingStorage != initialRemainingStorage { t.Fatal("manager is changing the amount of storage remaining when removing virtual sectors") } // Try to remove the sector using a wildcard expiry height. err = smt.sm.RemoveSector(root, expiryHeights[0]+548e6) if err != ErrSectorNotFound { t.Fatal(err) } // Remove the sector from the manager. err = smt.sm.RemoveSector(root, expiryHeights[0]) if err != nil { t.Fatal(err) } // Check that the filesystem is housing the correct number of // sectors. infos, err = ioutil.ReadDir(storageFolderOne) if err != nil { t.Fatal(err) } infos2, err = ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } bonus := 0 if len(expiryHeights) == 1 { // If this is the last expiry height, the sector is no longer // viritual and is being removed for real, so we need to // subtract it from the expected total number of sectors. bonus++ } if len(infos)+len(infos2) != 20-i-bonus { t.Fatal("sector count is incorrect while managing virtual sectors") } // Update the sector map to reflect the removed sector. if len(expiryHeights) > 1 { expiryHeights = expiryHeights[1:] sectorUsageMap[root] = expiryHeights } else { expiryHeights = nil delete(sectorUsageMap, root) } err = smt.sectorUsageCheck(sectorUsageMap) if err != nil { t.Fatal(err) } } // Do a probabilistic reset of the manager, to verify that the // persistence structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Check that the remaining storage is still the same. _, remainingStorage := smt.sm.capacity() if remainingStorage != initialRemainingStorage+modules.SectorSize { t.Fatal("manager incorrectly updated remaining space when deleting the final height for a sector") } } // Add a third storage folder. prevTotalStorage, prevRemainingStorage := smt.sm.capacity() storageFolderThree := filepath.Join(smt.persistDir, "hd3") err = os.Mkdir(storageFolderThree, 0700) if err != nil { t.Fatal(err) } err = smt.sm.AddStorageFolder(storageFolderThree, minimumStorageFolderSize*2) if err != nil { t.Fatal(err) } // Check that the total storage and remaining storage updated correctly. totalStorage, remainingStorage = smt.sm.capacity() if totalStorage != prevTotalStorage+minimumStorageFolderSize*2 || remainingStorage != prevRemainingStorage+minimumStorageFolderSize*2 { t.Fatal("storage folder sizes are not being updated correctly when new storage folders are added") } // Add sectors until the storage folders have no more capacity. _, remainingStorage = smt.sm.capacity() remainingSectors := remainingStorage / modules.SectorSize for i := uint64(0); i < remainingSectors; i++ { sectorRoot, sectorData, err := createSector() if err != nil { t.Fatal(err) } err = smt.sm.AddSector(sectorRoot, 36, sectorData) if err != nil { t.Fatal(err) } sectorUsageMap[sectorRoot] = []types.BlockHeight{36} } // Add another sector, which will not fit in the manager. sectorRoot, sectorData, err = createSector() if err != nil { t.Fatal(err) } err = smt.sm.AddSector(sectorRoot, 36, sectorData) if err != errInsufficientStorageForSector { t.Fatal(err) } // Do a probabilistic reset of the manager, to verify that the persistence // structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } _, remainingStorage = smt.sm.capacity() if remainingStorage >= modules.SectorSize { t.Error("remaining storage is reporting incorrect result - should report that there is not enough room for another sector") } err = smt.sectorUsageCheck(sectorUsageMap) if err != nil { t.Fatal(err) } // Check the filesystem. infos, err = ioutil.ReadDir(storageFolderOne) if err != nil { t.Fatal(err) } if len(infos) != 8 { t.Fatal("expecting 8 sectors in storage folder one") } infos, err = ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } if len(infos) != 24 { t.Fatal("expecting 24 sectors in storage folder two") } infos, err = ioutil.ReadDir(storageFolderThree) if err != nil { t.Fatal(err) } if len(infos) != 16 { t.Fatal("expecting 16 sectors in storage folder three") } // Do some resizing, to cause sectors to be moved around. Every storage // folder should have sectors that get moved off of it. err = smt.sm.ResizeStorageFolder(1, minimumStorageFolderSize*6) if err != nil { t.Fatal(err) } err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize) if err != nil { t.Fatal(err) } err = smt.sm.ResizeStorageFolder(2, minimumStorageFolderSize) if err != nil { t.Fatal(err) } err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize*6) if err != nil { t.Fatal(err) } err = smt.sm.ResizeStorageFolder(1, minimumStorageFolderSize) if err != nil { t.Fatal(err) } // Check that all storage folders are reporting successful reads and // writes, with no failures. for _, sf := range smt.sm.storageFolders { if sf.SuccessfulWrites <= 0 || sf.SuccessfulReads <= 0 || sf.FailedWrites > 0 || sf.FailedReads > 0 { t.Error("disk stats aren't making sense") } } // Remove all of the sectors. i := 0 for sectorRoot, expiryHeights := range sectorUsageMap { // Grab the initial remaining storage, to make sure that it's not being // changed when one instance of a repeated sector is removed. _, initialRemainingStorage := smt.sm.capacity() // Remove the heights one at a time. for j := range expiryHeights { // Check that the remaining storage is still the same. _, remainingStorage := smt.sm.capacity() if remainingStorage != initialRemainingStorage { t.Fatal("manager is changing the amount of storage remaining when removing virtual sectors") } // Remove the sector from the manager. err = smt.sm.RemoveSector(sectorRoot, expiryHeights[j]) if err != nil { t.Fatal(err) } // Check that the filesystem is housing the correct number of // sectors. infos, err := ioutil.ReadDir(storageFolderOne) if err != nil { t.Fatal(err) } infos2, err := ioutil.ReadDir(storageFolderTwo) if err != nil { t.Fatal(err) } infos3, err := ioutil.ReadDir(storageFolderThree) if err != nil { t.Fatal(err) } bonus := 0 if j == len(expiryHeights)-1 { // If this is the last expiry height, the sector is no longer // viritual and is being removed for real, so we need to // subtract it from the expected total number of sectors. bonus++ } if len(infos)+len(infos2)+len(infos3) != 48-i-bonus { t.Error(len(infos)+len(infos2)+len(infos3), i, bonus) t.Fatal("sector count is incorrect while managing virtual sectors") } } // Do a probabilistic reset of the manager, to verify that the // persistence structures can reboot without causing issues. err = smt.probabilisticReset() if err != nil { t.Fatal(err) } // Check that the remaining storage is still the same. _, remainingStorage := smt.sm.capacity() if remainingStorage != initialRemainingStorage+modules.SectorSize { t.Fatal("manager incorrectly updated remaining space when deleting the final height for a sector") } i++ } // Check that all storage folders have successful writes, and no failed // reads or writes. for _, sf := range smt.sm.storageFolders { if sf.SuccessfulWrites <= 0 || sf.SuccessfulReads <= 0 || sf.FailedWrites > 0 || sf.FailedReads > 0 { t.Error("disk stats aren't making sense") } } // Remove all of the storage folders. for i := 0; i < 3; i++ { err = smt.sm.RemoveStorageFolder(0, false) if err != nil { t.Fatal(err) } } // Check the filesystem, there should be 3 files in the manager folder // (storagemanager.db, storagemanager.json, storagemanager.log). // NOTE: on Windows, a lock file for the db will also be present. infos, err = ioutil.ReadDir(smt.sm.persistDir) if err != nil { t.Fatal(err) } if len(infos) != 3 && len(infos) != 4 { t.Error("unexpected number of files in the manager directory") } } Sia-1.0.4/modules/host/storagemanager/storagefolders_test.go000066400000000000000000000210021302033070500242510ustar00rootroot00000000000000package storagemanager import ( "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" ) // TestStorageFolderUIDString probes the uidString method of the storage // folder. func TestStorageFolderUIDString(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // Create a series of uid->string mappings that represent the expected // output of calling uidString on a storage folder. trials := []struct { uid []byte str string }{ { []byte{0}, "00", }, { []byte{255}, "ff", }, { []byte{50}, "32", }, { []byte{61}, "3d", }, { []byte{248}, "f8", }, } for _, trial := range trials { sf := &storageFolder{ UID: trial.uid, } str := sf.uidString() if str != trial.str { t.Error("failed UID string trial", str, sf.uidString()) } } } // TestStorageFolderUIDStringSanity probes the sanity checks of the uidString // method of the storage folder. func TestStorageFolderUIDStringSanity(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // Create a storage folder with an illegal UID size. sf := &storageFolder{ UID: []byte{0, 1}, } // Catch the resulting panic. defer func() { r := recover() if r == nil { t.Error("sanity check was not triggered upon incorrect usage of uidString") } }() _ = sf.uidString() } // TestAddStorageFolderUIDCollisions checks that storage folders can be added // with no risk of producing collisions in the storage folder UIDs. This test // relies on (explicitly checked) assumptions about the size of the name and // the number of allowed storage folders. func TestAddStorageFolderUIDCollisions(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() smt, err := newStorageManagerTester("TestAddStorageFolderUIDCollisions") if err != nil { t.Fatal(err) } defer smt.Close() // Check that the environment requirements for the test have been met. if storageFolderUIDSize != 1 { t.Fatal("For this test, the storage manager must be using storage folder UIDs that are 1 byte") } if maximumStorageFolders < 100 { t.Fatal("For this test, the storage manager must be allowed to have at least 100 storage folders") } // Create 100 storage folders, and check that there are no collisions // between any of them. Because the UID is only using 1 byte, once there // are more than 64 there will be at least 1/4 chance of a collision for // each randomly selected UID. Running into collisions is virtually // guaranteed, and running into repeated collisions (where two UIDs // consecutively collide with existing UIDs) are highly likely. for i := 0; i < maximumStorageFolders; i++ { err = smt.addRandFolder(minimumStorageFolderSize) if err != nil { t.Fatal(err) } } // Check that there are no collisions. uidMap := make(map[uint8]struct{}) for _, sf := range smt.sm.storageFolders { _, exists := uidMap[uint8(sf.UID[0])] if exists { t.Error("Collision") } uidMap[uint8(sf.UID[0])] = struct{}{} } // For coverage purposes, try adding a storage folder after the maximum // number of storage folders has been reached. err = smt.addRandFolder(minimumStorageFolderSize) if err != errMaxStorageFolders { t.Fatal("expecting errMaxStorageFolders:", err) } // Try again, this time removing a random storage folder and then adding // another one repeatedly - enough times to exceed the 256 possible folder // UIDs that can be chosen in the testing environment. for i := 0; i < 300; i++ { // Replace the very first storage folder. err = smt.sm.RemoveStorageFolder(0, false) if err != nil { t.Fatal(err) } err = smt.addRandFolder(minimumStorageFolderSize) if err != nil { t.Fatal(err) } // Replace a random storage folder. n, err := crypto.RandIntn(100) if err != nil { t.Fatal(err) } err = smt.sm.RemoveStorageFolder(n, false) if err != nil { t.Fatal(err) } err = smt.addRandFolder(minimumStorageFolderSize) if err != nil { t.Fatal(err) } } uidMap = make(map[uint8]struct{}) for _, sf := range smt.sm.storageFolders { _, exists := uidMap[uint8(sf.UID[0])] if exists { t.Error("Collision") } uidMap[uint8(sf.UID[0])] = struct{}{} } } // TestEmptiestStorageFolder checks that emptiestStorageFolder will correctly // select the emptiest storage folder out of a provided list of storage // folders. func TestEmptiestStorageFolder(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // Create a series of uid->string mappings that represent the expected // output of calling uidString on a storage folder. trials := []struct { folders []*storageFolder emptiestIndex int }{ // Empty input. { []*storageFolder{}, -1, }, // Single valid storage folder. { []*storageFolder{ { Size: minimumStorageFolderSize, SizeRemaining: minimumStorageFolderSize, }, }, 0, }, // Single full storage folder. { []*storageFolder{ { Size: minimumStorageFolderSize, SizeRemaining: 0, }, }, -1, }, // Single nearly full storage folder. { []*storageFolder{ { Size: minimumStorageFolderSize, SizeRemaining: modules.SectorSize - 1, }, }, -1, }, // Two valid storage folders, first is emptier. { []*storageFolder{ { Size: minimumStorageFolderSize, SizeRemaining: modules.SectorSize + 1, }, { Size: minimumStorageFolderSize, SizeRemaining: modules.SectorSize, }, }, 0, }, // Two valid storage folders, second is emptier. { []*storageFolder{ { Size: minimumStorageFolderSize, SizeRemaining: modules.SectorSize, }, { Size: minimumStorageFolderSize, SizeRemaining: modules.SectorSize + 1, }, }, 1, }, // Two valid storage folders, first is emptier by percentage but can't // hold a new sector. { []*storageFolder{ { Size: minimumStorageFolderSize, SizeRemaining: modules.SectorSize - 1, }, { Size: minimumStorageFolderSize * 5, SizeRemaining: modules.SectorSize, }, }, 1, }, // Two valid storage folders, first is emptier by volume but not // percentage. { []*storageFolder{ { Size: minimumStorageFolderSize * 4, SizeRemaining: modules.SectorSize * 2, }, { Size: minimumStorageFolderSize, SizeRemaining: modules.SectorSize, }, }, 1, }, // Two valid storage folders, second is emptier by volume but not // percentage. { []*storageFolder{ { Size: minimumStorageFolderSize, SizeRemaining: modules.SectorSize, }, { Size: minimumStorageFolderSize * 4, SizeRemaining: modules.SectorSize * 2, }, }, 0, }, // Three valid storage folders, second is emptier by percentage but not // volume. { []*storageFolder{ { Size: minimumStorageFolderSize * 4, SizeRemaining: modules.SectorSize * 2, }, { Size: minimumStorageFolderSize, SizeRemaining: modules.SectorSize, }, { Size: minimumStorageFolderSize * 4, SizeRemaining: modules.SectorSize * 2, }, }, 1, }, // Three storage folders, none have room for a sector. { []*storageFolder{ { Size: minimumStorageFolderSize * 4, SizeRemaining: modules.SectorSize - 1, }, { Size: minimumStorageFolderSize, SizeRemaining: 0, }, { Size: minimumStorageFolderSize * 4, SizeRemaining: 1, }, }, -1, }, } for i, trial := range trials { sf, index := emptiestStorageFolder(trial.folders) if index != trial.emptiestIndex { t.Error("trial", i, "index mismatch") } if index > 0 && sf != trial.folders[index] { t.Error("trial", i, "folder mismatch") } if index < 0 && sf != nil { t.Error("non-nil storage folder returned but there was no winner") } } } // TestRepeatStorageFolderPath checks that the host correctly rejects a storage // folder if there is already a storage folder linked to the same path. func TestRepeatStorageFolderPath(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() smt, err := newStorageManagerTester("TestRepeatStorageFolderPath") if err != nil { t.Fatal(err) } defer smt.Close() err = smt.sm.AddStorageFolder(smt.persistDir, minimumStorageFolderSize) if err != nil { t.Fatal(err) } err = smt.sm.AddStorageFolder(smt.persistDir, minimumStorageFolderSize) if err != ErrRepeatFolder { t.Fatal("expected ErrRepeatFolder, got", err) } } Sia-1.0.4/modules/host/storagemanager/storagemanager.go000066400000000000000000000112241302033070500231730ustar00rootroot00000000000000// package storagemanager implements a storage manager for the host on Sia. The // storage manager can add sectors, remove sectors, and manage how sectors are // distributed between a number of storage folders. package storagemanager // TODO: The documentation still talks like this code is a part of the host. import ( "errors" "path/filepath" "sync" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/persist" ) const ( // Names of the various persist files in the storage manager. dbFilename = "storagemanager.db" logFile = "storagemanager.log" settingsFile = "storagemanager.json" ) var ( // dbMetadata is the header that gets applied to the database to identify a // version and indicate what type of data is being stored in the database. dbMetadata = persist.Metadata{ Header: "Sia Storage Manager DB", Version: "0.6.0", } // persistMetadata is the header that gets added to the persist file to // identify the type of file and the version of the file. persistMetadata = persist.Metadata{ Header: "Sia Storage Manager", Version: "0.6.0", } errStorageManagerClosed = errors.New("call is disabled because storage manager is closed") ) // StorageManager tracks multiple storage folders, and is responsible for // adding sectors, removing sectors, and overall managing the way that data is // stored. type StorageManager struct { // Dependencies. dependencies // Storage management information. sectorSalt crypto.Hash storageFolders []*storageFolder // Utilities. db *persist.BoltDatabase log *persist.Logger mu sync.RWMutex persistDir string // The resource lock is held by threaded functions for the duration of // their operation. Functions should grab the resource lock as a read lock // unless they are planning on manipulating the 'closed' variable. // Readlocks are used so that multiple functions can use resources // simultaneously, but the resources are not closed until all functions // accessing them have returned. closed bool resourceLock sync.RWMutex } // Close will shut down the storage manager. func (sm *StorageManager) Close() (composedError error) { // Grab the resource lock and indicate that the host is closing. Concurrent // functions hold the resource lock until they terminate, meaning that no // threaded function will be running by the time the resource lock is // acquired. sm.resourceLock.Lock() closed := sm.closed sm.closed = true sm.resourceLock.Unlock() if closed { return nil } // Close the bolt database. err := sm.db.Close() if err != nil { composedError = composeErrors(composedError, err) } // Save the latest host state. sm.mu.Lock() err = sm.saveSync() sm.mu.Unlock() if err != nil { composedError = composeErrors(composedError, err) } // Close the logger. The logger should be the last thing to shut down so // that all other objects have access to logging while closing. err = sm.log.Close() if err != nil { composedError = composeErrors(composedError, err) } return composedError } // newStorageManager creates a new storage manager. func newStorageManager(dependencies dependencies, persistDir string) (*StorageManager, error) { sm := &StorageManager{ dependencies: dependencies, persistDir: persistDir, } // Create the perist directory if it does not yet exist. err := dependencies.mkdirAll(sm.persistDir, 0700) if err != nil { return nil, err } // Initialize the logger. Logging should be initialized ASAP, because the // rest of the initialization makes use of the logger. sm.log, err = dependencies.newLogger(filepath.Join(sm.persistDir, logFile)) if err != nil { return nil, err } // Open the database containing the host's storage obligation metadata. sm.db, err = dependencies.openDatabase(dbMetadata, filepath.Join(sm.persistDir, dbFilename)) if err != nil { // An error will be returned if the database has the wrong version, but // as of writing there was only one version of the database and all // other databases would be incompatible. _ = sm.log.Close() return nil, err } // After opening the database, it must be initialized. Most commonly, // nothing happens. But for new databases, a set of buckets must be // created. Initialization is also a good time to run sanity checks. err = sm.initDB() if err != nil { _ = sm.log.Close() _ = sm.db.Close() return nil, err } // Load the prior persistence structures. err = sm.load() if err != nil { _ = sm.log.Close() _ = sm.db.Close() return nil, err } return sm, nil } // New returns an initialized StorageManager. func New(persistDir string) (*StorageManager, error) { return newStorageManager(productionDependencies{}, persistDir) } Sia-1.0.4/modules/host/storagemanager/storagemanager_test.go000066400000000000000000000060661302033070500242420ustar00rootroot00000000000000package storagemanager import ( "os" "path/filepath" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" ) // storageManagerTester holds a testing-initialized storage manager and any // additional fields that may be useful while testing. type storageManagerTester struct { sm *StorageManager persistDir string } // addRandFolder connects a storage folder to a random directory in the // tester's persist dir. func (smt *storageManagerTester) addRandFolder(size uint64) error { dir := filepath.Join(smt.persistDir, persist.RandomSuffix()) err := os.Mkdir(dir, 0700) if err != nil { return err } return smt.sm.AddStorageFolder(dir, size) } // capacity returns the amount of storage still available on the machine. The // amount can be negative if the total capacity was reduced to below the active // capacity. func (sm *StorageManager) capacity() (total, remaining uint64) { // Total storage can be computed by summing the size of all the storage // folders. for _, sf := range sm.storageFolders { total += sf.Size remaining += sf.SizeRemaining } return total, remaining } // createSector makes a random, unique sector that can be inserted into the // storage manager. func createSector() (sectorRoot crypto.Hash, sectorData []byte, err error) { sectorData, err = crypto.RandBytes(int(modules.SectorSize)) if err != nil { return crypto.Hash{}, nil, err } sectorRoot = crypto.MerkleRoot(sectorData) return sectorRoot, sectorData, nil } // newStorageManagerTester creates a storage tester ready for use. func newStorageManagerTester(name string) (*storageManagerTester, error) { testdir := build.TempDir(modules.StorageManagerDir, name) sm, err := New(filepath.Join(testdir, modules.StorageManagerDir)) if err != nil { return nil, err } smt := &storageManagerTester{ sm: sm, persistDir: testdir, } return smt, nil } // probabilisticReset will probabilistically reboot the storage manager before // continuing. This helps to verify that the persistence is working correctly. // The reset is probabilistic to make sure that the test is not passing because // of the reset. func (smt *storageManagerTester) probabilisticReset() error { rand, err := crypto.RandIntn(3) if err != nil { return err } if rand == 1 { // Grab the potentially faulty dependencies and replace them with good // dependencies so that closing happens without issues. deps := smt.sm.dependencies smt.sm.dependencies = productionDependencies{} // Close the storage manager, then create a new storage manager to // replace it. err = smt.sm.Close() if err != nil { return err } // Open the storage manager with production dependencies so that there // are no errors. sm, err := New(filepath.Join(smt.persistDir, modules.StorageManagerDir)) if err != nil { return err } sm.dependencies = deps smt.sm = sm } return nil } // Close shuts down all of the components of the storage manager tester. func (smt *storageManagerTester) Close() error { return smt.sm.Close() } Sia-1.0.4/modules/host/storageobligations.go000066400000000000000000001151021302033070500210740ustar00rootroot00000000000000package host // storageobligations.go is responsible for managing the storage obligations // within the host - making sure that any file contracts, transaction // dependencies, file contract revisions, and storage proofs are making it into // the blockchain in a reasonable time. // // NOTE: Currently, the code partially supports changing the storage proof // window in file contract revisions, however the action item code will not // handle it correctly. Until the action item code is improved (to also handle // byzantine situations where the renter submits prior revisions), the host // should not support changing the storage proof window, especially to further // in the future. // TODO: Need to queue the action item for checking on the submission status of // the file contract revision. Also need to make sure that multiple actions are // being taken if needed. // TODO: Make sure that the origin tranasction set is not submitted to the // transaction pool before addSO is called - if it is, there will be a // duplicate transaction error, and then the storage obligation will return an // error, which is bad. Well, or perhas we just need to have better logic // handling. // TODO: Need to make sure that 'revision confirmed' is actually looking only // at the most recent revision (I think it is...) // TODO: Make sure that not too many action items are being created. import ( "encoding/binary" "encoding/json" "errors" "sync" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) const ( obligationUnresolved storageObligationStatus = iota // Indicatees that an unitialized value was used. obligationRejected // Indicates that the obligation never got started, no revenue gained or lost. obligationSucceeded // Indicates that the obligation was completed, revenues were gained. obligationFailed // Indicates that the obligation failed, revenues and collateral were lost. ) var ( // errDuplicateStorageObligation is returned when the storage obligation // database already has a storage obligation with the provided file // contract. This error should only happen in the event of a developer // mistake. errDuplicateStorageObligation = errors.New("storage obligation has a file contract which conflicts with an existing storage obligation") // errInsaneFileContractOutputCounts is returned when a file contract has // the wrong number of outputs for either the valid or missed payouts. errInsaneFileContractOutputCounts = errors.New("file contract has incorrect number of outputs for the valid or missed payouts") // errInsaneFileContractRevisionOutputCounts is returned when a file // contract has the wrong number of outputs for either the valid or missed // payouts. errInsaneFileContractRevisionOutputCounts = errors.New("file contract revision has incorrect number of outputs for the valid or missed payouts") // errInsaneOriginSetFileContract is returned is the final transaction of // the origin transaction set of a storage obligation does not have a file // contract in the final transaction - there should be a file contract // associated with every storage obligation. errInsaneOriginSetFileContract = errors.New("origin transaction set of storage obligation should have one file contract in the final transaction") // errInsaneOriginSetSize is returned if the origin transaction set of a // storage obligation is empty - there should be a file contract associated // with every storage obligation. errInsaneOriginSetSize = errors.New("origin transaction set of storage obligation is size zero") // errInsaneRevisionSetRevisionCount is returned if the final transaction // in the revision transaction set of a storage obligation has more or less // than one file contract revision. errInsaneRevisionSetRevisionCount = errors.New("revision transaction set of storage obligation should have one file contract revision in the final transaction") // errInsaneStorageObligationRevision is returned if there is an attempted // storage obligation revision which does not have sensical inputs. errInsaneStorageObligationRevision = errors.New("revision to storage obligation does not make sense") // errInsaneStorageObligationRevisionData is returned if there is an // attempted storage obligation revision which does not have sensical // inputs. errInsaneStorageObligationRevisionData = errors.New("revision to storage obligation has insane data") // errObligationUnlocked is returned when a storage obligation is being // removed from lock, but is already unlocked. errObligationUnlocked = errors.New("storage obligation is unlocked, and should not be getting unlocked") // errNoBuffer is returned if there is an attempted storage obligation that // needs to have the storage proof submitted in less than // revisionSubmissionBuffer blocks. errNoBuffer = errors.New("file contract rejected because storage proof window is too close") // errNoStorageObligation is returned if the requested storage obligation // is not found in the database. errNoStorageObligation = errors.New("storage obligation not found in database") ) type storageObligationStatus uint64 // storageObligation contains all of the metadata related to a file contract // and the storage contained by the file contract. type storageObligation struct { // Storage obligations are broken up into ordered atomic sectors that are // exactly 4MiB each. By saving the roots of each sector, storage proofs // and modifications to the data can be made inexpensively by making use of // the merkletree.CachedTree. Sectors can be appended, modified, or deleted // and the host can recompute the Merkle root of the whole file without // much computational or I/O expense. SectorRoots []crypto.Hash // Variables about the file contract that enforces the storage obligation. // The origin an revision transaction are stored as a set, where the set // contains potentially unconfirmed transactions. ContractCost types.Currency LockedCollateral types.Currency PotentialDownloadRevenue types.Currency PotentialStorageRevenue types.Currency PotentialUploadRevenue types.Currency RiskedCollateral types.Currency TransactionFeesAdded types.Currency OriginTransactionSet []types.Transaction RevisionTransactionSet []types.Transaction // Variables indicating whether the critical transactions in a storage // obligation have been confirmed on the blockchain. OriginConfirmed bool RevisionConfirmed bool ProofConfirmed bool ObligationStatus storageObligationStatus } // getStorageObligation fetches a storage obligation from the database tx. func getStorageObligation(tx *bolt.Tx, soid types.FileContractID) (so storageObligation, err error) { soBytes := tx.Bucket(bucketStorageObligations).Get(soid[:]) if soBytes == nil { return storageObligation{}, errNoStorageObligation } err = json.Unmarshal(soBytes, &so) if err != nil { return storageObligation{}, err } return so, nil } // putStorageObligation places a storage obligation into the database, // overwriting the existing storage obligation if there is one. func putStorageObligation(tx *bolt.Tx, so storageObligation) error { soBytes, err := json.Marshal(so) if err != nil { return err } soid := so.id() return tx.Bucket(bucketStorageObligations).Put(soid[:], soBytes) } // expiration returns the height at which the storage obligation expires. func (so storageObligation) expiration() types.BlockHeight { if len(so.RevisionTransactionSet) > 0 { return so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0].NewWindowStart } return so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContracts[0].WindowStart } // fileSize returns the size of the data protected by the obligation. func (so storageObligation) fileSize() uint64 { if len(so.RevisionTransactionSet) > 0 { return so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0].NewFileSize } return so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContracts[0].FileSize } // id returns the id of the storage obligation, which is definied by the file // contract id of the file contract that governs the storage contract. func (so storageObligation) id() types.FileContractID { return so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContractID(0) } // isSane checks that required assumptions about the storage obligation are // correct. func (so storageObligation) isSane() error { // There should be an origin transaction set. if len(so.OriginTransactionSet) == 0 { build.Critical("origin transaction set is empty") return errInsaneOriginSetSize } // The final transaction of the origin transaction set should have one file // contract. final := len(so.OriginTransactionSet) - 1 fcCount := len(so.OriginTransactionSet[final].FileContracts) if fcCount != 1 { build.Critical("wrong number of file contracts associated with storage obligation:", fcCount) return errInsaneOriginSetFileContract } // The file contract in the final transaction of the origin transaction set // should have two valid proof outputs and two missed proof outputs. lenVPOs := len(so.OriginTransactionSet[final].FileContracts[0].ValidProofOutputs) lenMPOs := len(so.OriginTransactionSet[final].FileContracts[0].MissedProofOutputs) if lenVPOs != 2 || lenMPOs != 2 { build.Critical("file contract has wrong number of VPOs and MPOs, expecting 2 each:", lenVPOs, lenMPOs) return errInsaneFileContractOutputCounts } // If there is a revision transaction set, there should be one file // contract revision in the final transaction. if len(so.RevisionTransactionSet) > 0 { final = len(so.OriginTransactionSet) - 1 fcrCount := len(so.OriginTransactionSet[final].FileContractRevisions) if fcrCount != 1 { build.Critical("wrong number of file contract revisions in final transaction of revision transaction set:", fcrCount) return errInsaneRevisionSetRevisionCount } // The file contract revision in the final transaction of the revision // transaction set should have two valid proof outputs and two missed // proof outputs. lenVPOs = len(so.RevisionTransactionSet[final].FileContractRevisions[0].NewValidProofOutputs) lenMPOs = len(so.RevisionTransactionSet[final].FileContractRevisions[0].NewMissedProofOutputs) if lenVPOs != 2 || lenMPOs != 2 { build.Critical("file contract has wrong number of VPOs and MPOs, expecting 2 each:", lenVPOs, lenMPOs) return errInsaneFileContractRevisionOutputCounts } } return nil } // merkleRoot returns the file merkle root of a storage obligation. func (so storageObligation) merkleRoot() crypto.Hash { if len(so.RevisionTransactionSet) > 0 { return so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0].NewFileMerkleRoot } return so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContracts[0].FileMerkleRoot } // payous returns the set of valid payouts and missed payouts that represent // the latest revision for the storage obligation. func (so storageObligation) payouts() (valid []types.SiacoinOutput, missed []types.SiacoinOutput) { valid = make([]types.SiacoinOutput, 2) missed = make([]types.SiacoinOutput, 2) if len(so.RevisionTransactionSet) > 0 { copy(valid, so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0].NewValidProofOutputs) copy(missed, so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0].NewMissedProofOutputs) return } copy(valid, so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContracts[0].ValidProofOutputs) copy(missed, so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContracts[0].MissedProofOutputs) return } // proofDeadline returns the height by which the storage proof must be // submitted. func (so storageObligation) proofDeadline() types.BlockHeight { if len(so.RevisionTransactionSet) > 0 { return so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0].NewWindowEnd } return so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContracts[0].WindowEnd } // value returns the value of fulfilling the storage obligation to the host. func (so storageObligation) value() types.Currency { return so.ContractCost.Add(so.PotentialDownloadRevenue).Add(so.PotentialStorageRevenue).Add(so.PotentialUploadRevenue).Add(so.RiskedCollateral) } // queueActionItem adds an action item to the host at the input height so that // the host knows to perform maintenance on the associated storage obligation // when that height is reached. func (h *Host) queueActionItem(height types.BlockHeight, id types.FileContractID) error { // Sanity check - action item should be at a higher height than the current // block height. if height <= h.blockHeight { h.log.Critical("action item queued improperly") } return h.db.Update(func(tx *bolt.Tx) error { // Translate the height into a byte slice. heightBytes := make([]byte, 8) binary.BigEndian.PutUint64(heightBytes, uint64(height)) // Get the list of action items already at this height and extend it. bai := tx.Bucket(bucketActionItems) existingItems := bai.Get(heightBytes) var extendedItems = make([]byte, len(existingItems), len(existingItems)+len(id[:])) copy(extendedItems, existingItems) extendedItems = append(extendedItems, id[:]...) return bai.Put(heightBytes, extendedItems) }) } // addStorageObligation adds a storage obligation to the host. Because this // operation can return errors, the transactions should not be submitted to the // blockchain until after this function has indicated success. All of the // sectors that are present in the storage obligation should already be on // disk, which means that addStorageObligation should be exclusively called // when creating a new, empty file contract or when renewing an existing file // contract. func (h *Host) addStorageObligation(so storageObligation) error { // Sanity check - obligation should be under lock while being added. soid := so.id() _, exists := h.lockedStorageObligations[soid] if !exists { h.log.Critical("addStorageObligation called with an obligation that is not locked") } // Sanity check - There needs to be enough time left on the file contract // for the host to safely submit the file contract revision. if h.blockHeight+revisionSubmissionBuffer >= so.expiration() { h.log.Critical("submission window was not verified before trying to submit a storage obligation") return errNoBuffer } // Sanity check - the resubmission timeout needs to be smaller than storage // proof window. if so.expiration()+resubmissionTimeout >= so.proofDeadline() { h.log.Critical("host is misconfigured - the storage proof window needs to be long enough to resubmit if needed") return errors.New("fill me in") } // Add the storage obligation information to the database. err := h.db.Update(func(tx *bolt.Tx) error { // Sanity check - a storage obligation using the same file contract id // should not already exist. This situation can happen if the // transaction pool ejects a file contract and then a new one is // created. Though the file contract will have the same terms, some // other conditions might cause problems. The check for duplicate file // contract ids should happen during the negotiation phase, and not // during the 'addStorageObligation' phase. bso := tx.Bucket(bucketStorageObligations) // If the storage obligation already has sectors, it means that the // file contract is being renewed, and that the sector should be // re-added with a new expriation height. If there is an error at any // point, all of the sectors should be removed. if len(so.SectorRoots) != 0 { err := h.AddSectorBatch(so.SectorRoots, so.expiration()) if err != nil { return err } } // Add the storage obligation to the database. soBytes, err := json.Marshal(so) if err != nil { return err } return bso.Put(soid[:], soBytes) }) if err != nil { return err } // Update the host financial metrics with regards to this storage // obligation. h.financialMetrics.ContractCount++ h.financialMetrics.PotentialContractCompensation = h.financialMetrics.PotentialContractCompensation.Add(so.ContractCost) h.financialMetrics.LockedStorageCollateral = h.financialMetrics.LockedStorageCollateral.Add(so.LockedCollateral) h.financialMetrics.PotentialStorageRevenue = h.financialMetrics.PotentialStorageRevenue.Add(so.PotentialStorageRevenue) h.financialMetrics.PotentialDownloadBandwidthRevenue = h.financialMetrics.PotentialDownloadBandwidthRevenue.Add(so.PotentialDownloadRevenue) h.financialMetrics.PotentialUploadBandwidthRevenue = h.financialMetrics.PotentialUploadBandwidthRevenue.Add(so.PotentialUploadRevenue) h.financialMetrics.RiskedStorageCollateral = h.financialMetrics.RiskedStorageCollateral.Add(so.RiskedCollateral) h.financialMetrics.TransactionFeeExpenses = h.financialMetrics.TransactionFeeExpenses.Add(so.TransactionFeesAdded) // Set an action item that will have the host verify that the file contract // has been submitted to the blockchain, then another to submit the file // contract revision to the blockchain, and another to submit the storage // proof. err0 := h.tpool.AcceptTransactionSet(so.OriginTransactionSet) if err0 != nil { h.log.Println("Failed to add storage obligation, transaction set was not accepted:", err0) } // The file contract was already submitted to the blockchain, need to check // after the resubmission timeout that it was submitted successfully. err1 := h.queueActionItem(h.blockHeight+resubmissionTimeout, soid) // Queue an action item to submit the file contract revision - if there is // never a file contract revision, the handling of this action item will be // a no-op. err2 := h.queueActionItem(so.expiration()-revisionSubmissionBuffer, soid) // The storage proof should be submitted err3 := h.queueActionItem(so.expiration()+resubmissionTimeout, soid) err = composeErrors(err0, err1, err2, err3) if err != nil { h.log.Println("Error with transaction set, redacting obligation, id", so.id()) return composeErrors(err, h.removeStorageObligation(so, obligationRejected)) } return nil } // modifyStorageObligation will take an updated storage obligation along with a // list of sector changes and update the database to account for all of it. The // sector modifications are only used to update the sector database, they will // not be used to modify the storage obligation (most importantly, this means // that sectorRoots needs to be updated by the calling function). Virtual // sectors will be removed the number of times that they are listed, to remove // multiple instances of the same virtual sector, the virtural sector will need // to appear in 'sectorsRemoved' multiple times. Same with 'sectorsGained'. func (h *Host) modifyStorageObligation(so storageObligation, sectorsRemoved []crypto.Hash, sectorsGained []crypto.Hash, gainedSectorData [][]byte) error { // Sanity check - obligation should be under lock while being modified. soid := so.id() _, exists := h.lockedStorageObligations[soid] if !exists { h.log.Critical("modifyStorageObligation called with an obligation that is not locked") } // Sanity check - there needs to be enough time to submit the file contract // revision to the blockchain. if so.expiration()-revisionSubmissionBuffer <= h.blockHeight { h.log.Critical("revision submission window was not verified before trying to modify a storage obligation") return errNoBuffer } // Sanity check - sectorsGained and gainedSectorData need to have the same length. if len(sectorsGained) != len(gainedSectorData) { h.log.Critical("modifying a revision with garbage sector data", len(sectorsGained), len(gainedSectorData)) return errInsaneStorageObligationRevision } // Sanity check - all of the sector data should be modules.SectorSize for _, data := range gainedSectorData { if uint64(len(data)) != modules.SectorSize { h.log.Critical("modifying a revision with garbase sector sizes", len(data)) return errInsaneStorageObligationRevision } } // Note, for safe error handling, the operation order should be: add // sectors, update database, remove sectors. If the adding or update fails, // the added sectors should be removed and the storage obligation shoud be // considered invalid. If the removing fails, this is okay, it's ignored // and left to consistency checks and user actions to fix (will reduce host // capacity, but will not inhibit the host's ability to submit storage // proofs) var i int var err error for i = range sectorsGained { err = h.AddSector(sectorsGained[i], so.expiration(), gainedSectorData[i]) if err != nil { break } } if err != nil { // Because there was an error, all of the sectors that got added need // to be reverted. for j := 0; j < i; j++ { // Error is not checked because there's nothing useful that can be // done about an error. _ = h.RemoveSector(sectorsGained[j], so.expiration()) } return err } // Update the database to contain the new storage obligation. var oldSO storageObligation err = h.db.Update(func(tx *bolt.Tx) error { // Get the old storage obligation as a reference to know how to upate // the host financial stats. oldSO, err = getStorageObligation(tx, soid) if err != nil { return err } // Store the new storage obligation to replace the old one. return putStorageObligation(tx, so) }) if err != nil { // Because there was an error, all of the sectors that got added need // to be reverted. for i := range sectorsGained { // Error is not checked because there's nothing useful that can be // done about an error. _ = h.RemoveSector(sectorsGained[i], so.expiration()) } return err } // Call removeSector for all of the sectors that have been removed. for k := range sectorsRemoved { // Error is not checkeed because there's nothing useful that can be // done about an error. Failing to remove a sector is not a terrible // place to be, especially if the host can run consistency checks. _ = h.RemoveSector(sectorsRemoved[k], so.expiration()) } // Update the financial information for the storage obligation - remove the // old values. h.financialMetrics.PotentialContractCompensation = h.financialMetrics.PotentialContractCompensation.Sub(oldSO.ContractCost) h.financialMetrics.LockedStorageCollateral = h.financialMetrics.LockedStorageCollateral.Sub(oldSO.LockedCollateral) h.financialMetrics.PotentialStorageRevenue = h.financialMetrics.PotentialStorageRevenue.Sub(oldSO.PotentialStorageRevenue) h.financialMetrics.PotentialDownloadBandwidthRevenue = h.financialMetrics.PotentialDownloadBandwidthRevenue.Sub(oldSO.PotentialDownloadRevenue) h.financialMetrics.PotentialUploadBandwidthRevenue = h.financialMetrics.PotentialUploadBandwidthRevenue.Sub(oldSO.PotentialUploadRevenue) h.financialMetrics.RiskedStorageCollateral = h.financialMetrics.RiskedStorageCollateral.Sub(oldSO.RiskedCollateral) h.financialMetrics.TransactionFeeExpenses = h.financialMetrics.TransactionFeeExpenses.Sub(oldSO.TransactionFeesAdded) // Update the financial information for the storage obligation - apply the // new values. h.financialMetrics.PotentialContractCompensation = h.financialMetrics.PotentialContractCompensation.Add(so.ContractCost) h.financialMetrics.LockedStorageCollateral = h.financialMetrics.LockedStorageCollateral.Add(so.LockedCollateral) h.financialMetrics.PotentialStorageRevenue = h.financialMetrics.PotentialStorageRevenue.Add(so.PotentialStorageRevenue) h.financialMetrics.PotentialDownloadBandwidthRevenue = h.financialMetrics.PotentialDownloadBandwidthRevenue.Add(so.PotentialDownloadRevenue) h.financialMetrics.PotentialUploadBandwidthRevenue = h.financialMetrics.PotentialUploadBandwidthRevenue.Add(so.PotentialUploadRevenue) h.financialMetrics.RiskedStorageCollateral = h.financialMetrics.RiskedStorageCollateral.Add(so.RiskedCollateral) h.financialMetrics.TransactionFeeExpenses = h.financialMetrics.TransactionFeeExpenses.Add(so.TransactionFeesAdded) return nil } // removeStorageObligation will remove a storage obligation from the host, // either due to failure or success. func (h *Host) removeStorageObligation(so storageObligation, sos storageObligationStatus) error { // Call removeSector for every sector in the storage obligation. for _, root := range so.SectorRoots { // Error is not checked, we want to call remove on every sector even if // there are problems - disk health information will be updated. _ = h.RemoveSector(root, so.expiration()) } // Update the host revenue metrics based on the status of the obligation. if sos == obligationUnresolved { h.log.Critical("storage obligation 'unresolved' during call to removeStorageObligation, id", so.id()) } if sos == obligationRejected { if h.financialMetrics.TransactionFeeExpenses.Cmp(so.TransactionFeesAdded) >= 0 { h.financialMetrics.TransactionFeeExpenses = h.financialMetrics.TransactionFeeExpenses.Sub(so.TransactionFeesAdded) // Remove the obligation statistics as potential risk and income. h.log.Printf("Rejecting storage obligation expiring at block %v, current height is %v. Potential revenue is %v.\n", so.expiration(), h.blockHeight, h.financialMetrics.PotentialContractCompensation.Add(h.financialMetrics.PotentialStorageRevenue).Add(h.financialMetrics.PotentialDownloadBandwidthRevenue).Add(h.financialMetrics.PotentialUploadBandwidthRevenue)) h.financialMetrics.PotentialContractCompensation = h.financialMetrics.PotentialContractCompensation.Sub(so.ContractCost) h.financialMetrics.LockedStorageCollateral = h.financialMetrics.LockedStorageCollateral.Sub(so.LockedCollateral) h.financialMetrics.PotentialStorageRevenue = h.financialMetrics.PotentialStorageRevenue.Sub(so.PotentialStorageRevenue) h.financialMetrics.PotentialDownloadBandwidthRevenue = h.financialMetrics.PotentialDownloadBandwidthRevenue.Sub(so.PotentialDownloadRevenue) h.financialMetrics.PotentialUploadBandwidthRevenue = h.financialMetrics.PotentialUploadBandwidthRevenue.Sub(so.PotentialUploadRevenue) h.financialMetrics.RiskedStorageCollateral = h.financialMetrics.RiskedStorageCollateral.Sub(so.RiskedCollateral) } } if sos == obligationSucceeded { // Remove the obligation statistics as potential risk and income. h.log.Printf("Succesfully submitted a storage proof. Revenue is %v.\n", h.financialMetrics.PotentialContractCompensation.Add(h.financialMetrics.PotentialStorageRevenue).Add(h.financialMetrics.PotentialDownloadBandwidthRevenue).Add(h.financialMetrics.PotentialUploadBandwidthRevenue)) h.financialMetrics.PotentialContractCompensation = h.financialMetrics.PotentialContractCompensation.Sub(so.ContractCost) h.financialMetrics.LockedStorageCollateral = h.financialMetrics.LockedStorageCollateral.Sub(so.LockedCollateral) h.financialMetrics.PotentialStorageRevenue = h.financialMetrics.PotentialStorageRevenue.Sub(so.PotentialStorageRevenue) h.financialMetrics.PotentialDownloadBandwidthRevenue = h.financialMetrics.PotentialDownloadBandwidthRevenue.Sub(so.PotentialDownloadRevenue) h.financialMetrics.PotentialUploadBandwidthRevenue = h.financialMetrics.PotentialUploadBandwidthRevenue.Sub(so.PotentialUploadRevenue) h.financialMetrics.RiskedStorageCollateral = h.financialMetrics.RiskedStorageCollateral.Sub(so.RiskedCollateral) // Add the obligation statistics as actual income. h.financialMetrics.ContractCompensation = h.financialMetrics.ContractCompensation.Add(so.ContractCost) h.financialMetrics.StorageRevenue = h.financialMetrics.StorageRevenue.Add(so.PotentialStorageRevenue) h.financialMetrics.DownloadBandwidthRevenue = h.financialMetrics.DownloadBandwidthRevenue.Add(so.PotentialDownloadRevenue) h.financialMetrics.UploadBandwidthRevenue = h.financialMetrics.UploadBandwidthRevenue.Add(so.PotentialUploadRevenue) } if sos == obligationFailed { // Remove the obligation statistics as potential risk and income. h.log.Printf("Missed storage proof. Revenue would have been %v.\n", h.financialMetrics.PotentialContractCompensation.Add(h.financialMetrics.PotentialStorageRevenue).Add(h.financialMetrics.PotentialDownloadBandwidthRevenue).Add(h.financialMetrics.PotentialUploadBandwidthRevenue)) h.financialMetrics.PotentialContractCompensation = h.financialMetrics.PotentialContractCompensation.Sub(so.ContractCost) h.financialMetrics.LockedStorageCollateral = h.financialMetrics.LockedStorageCollateral.Sub(so.LockedCollateral) h.financialMetrics.PotentialStorageRevenue = h.financialMetrics.PotentialStorageRevenue.Sub(so.PotentialStorageRevenue) h.financialMetrics.PotentialDownloadBandwidthRevenue = h.financialMetrics.PotentialDownloadBandwidthRevenue.Sub(so.PotentialDownloadRevenue) h.financialMetrics.PotentialUploadBandwidthRevenue = h.financialMetrics.PotentialUploadBandwidthRevenue.Sub(so.PotentialUploadRevenue) h.financialMetrics.RiskedStorageCollateral = h.financialMetrics.RiskedStorageCollateral.Sub(so.RiskedCollateral) // Add the obligation statistics as loss. h.financialMetrics.LostStorageCollateral = h.financialMetrics.LostStorageCollateral.Add(so.RiskedCollateral) h.financialMetrics.LostRevenue = h.financialMetrics.LostRevenue.Add(so.ContractCost).Add(so.PotentialStorageRevenue).Add(so.PotentialDownloadRevenue).Add(so.PotentialUploadRevenue) } // Update the storage obligation to be finalized but still in-database. The // obligation status is updated so that the user can see how the obligation // ended up, and the sector roots are removed because they are large // objects with little purpose once storage proofs are no longer needed. h.financialMetrics.ContractCount-- so.ObligationStatus = sos so.SectorRoots = nil return h.db.Update(func(tx *bolt.Tx) error { return putStorageObligation(tx, so) }) } // threadedHandleActionItem will look at a storage obligation and determine // which action is necessary for the storage obligation to succeed. func (h *Host) threadedHandleActionItem(soid types.FileContractID, wg *sync.WaitGroup) { // The calling thread is responsible for calling Add to the thread group. defer wg.Done() // Lock the storage obligation in question. h.managedLockStorageObligation(soid) defer func() { h.managedUnlockStorageObligation(soid) }() // Convert the storage obligation id into a storage obligation. var err error var so storageObligation h.mu.RLock() blockHeight := h.blockHeight err = h.db.View(func(tx *bolt.Tx) error { so, err = getStorageObligation(tx, soid) return err }) h.mu.RUnlock() if err != nil { h.log.Println("Could not get storage obligation:", err) return } // Check whether the storage obligation has already been completed. if so.ObligationStatus != obligationUnresolved { // Storage obligation has already been completed, skip action item. return } // Check whether the file contract has been seen. If not, resubmit and // queue another action item. Check for death. (signature should have a // kill height) if !so.OriginConfirmed { // Submit the transaction set again, try to get the transaction // confirmed. err := h.tpool.AcceptTransactionSet(so.OriginTransactionSet) if err != nil { h.log.Debugln("Could not get origin transaction set accepted", err) // Check if the transaction is invalid with the current consensus set. // If so, the transaction is highly unlikely to ever be confirmed, and // the storage obligation should be removed. This check should come // after logging the errror so that the function can quit. // // TODO: If the host or tpool is behind consensus, might be difficult // to have certainty about the issue. If some but not all of the // parents are confirmed, might be some difficulty. _, t := err.(modules.ConsensusConflict) if t { h.log.Println("Consensus conflict on the origin transaction set, id", so.id()) h.mu.Lock() err = h.removeStorageObligation(so, obligationRejected) h.mu.Unlock() if err != nil { h.log.Println("Error removing storage obligation:", err) } return } } // Queue another action item to check the status of the transaction. h.mu.Lock() err = h.queueActionItem(h.blockHeight+resubmissionTimeout, so.id()) h.mu.Unlock() if err != nil { h.log.Println("Error queuing action item:", err) } } // Check if the file contract revision is ready for submission. Check for death. if !so.RevisionConfirmed && len(so.RevisionTransactionSet) > 0 && blockHeight > so.expiration()-revisionSubmissionBuffer { // Sanity check - there should be a file contract revision. rtsLen := len(so.RevisionTransactionSet) if rtsLen < 1 || len(so.RevisionTransactionSet[rtsLen-1].FileContractRevisions) != 1 { h.log.Critical("transaction revision marked as unconfirmed, yet there is no transaction revision") return } // Check if the revision has failed to submit correctly. if blockHeight > so.expiration() { // TODO: Check this error. // // TODO: this is not quite right, because a previous revision may // be confirmed, and the origin transaction may be confirmed, which // would confuse the revenue stuff a bit. Might happen frequently // due to the dynamic fee pool. h.log.Println("Full time has elapsed, but the revision transaction could not be submitted to consensus, id", so.id()) h.mu.Lock() h.removeStorageObligation(so, obligationRejected) h.mu.Unlock() return } // Queue another action item to check the status of the transaction. h.mu.Lock() err := h.queueActionItem(blockHeight+resubmissionTimeout, so.id()) h.mu.Unlock() if err != nil { h.log.Println("Error queuing action item:", err) } // Add a miner fee to the transaction and submit it to the blockchain. revisionTxnIndex := len(so.RevisionTransactionSet) - 1 revisionParents := so.RevisionTransactionSet[:revisionTxnIndex] revisionTxn := so.RevisionTransactionSet[revisionTxnIndex] builder := h.wallet.RegisterTransaction(revisionTxn, revisionParents) _, feeRecommendation := h.tpool.FeeEstimation() if so.value().Div64(2).Cmp(feeRecommendation) < 0 { // There's no sense submitting the revision if the fee is more than // half of the anticipated revenue - fee market went up // unexpectedly, and the money that the renter paid to cover the // fees is no longer enough. return } txnSize := uint64(len(encoding.MarshalAll(so.RevisionTransactionSet)) + 300) requiredFee := feeRecommendation.Mul64(txnSize) err = builder.FundSiacoins(requiredFee) if err != nil { h.log.Println("Error funding transaction fees", err) } builder.AddMinerFee(requiredFee) if err != nil { h.log.Println("Error adding miner fees", err) } feeAddedRevisionTransactionSet, err := builder.Sign(true) if err != nil { h.log.Println("Error signing transaction", err) } err = h.tpool.AcceptTransactionSet(feeAddedRevisionTransactionSet) if err != nil { h.log.Println("Error submitting transaction to transaction pool", err) } so.TransactionFeesAdded = so.TransactionFeesAdded.Add(requiredFee) // return } // Check whether a storage proof is ready to be provided, and whether it // has been accepted. Check for death. if !so.ProofConfirmed && blockHeight >= so.expiration()+resubmissionTimeout { h.log.Debugln("Host is attempting a storage proof for", so.id()) // If the window has closed, the host has failed and the obligation can // be removed. if so.proofDeadline() < blockHeight || len(so.SectorRoots) == 0 { h.log.Debugln("storage proof not confirmed by deadline, id", so.id()) h.mu.Lock() err := h.removeStorageObligation(so, obligationFailed) h.mu.Unlock() if err != nil { h.log.Println("Error removing storage obligation:", err) } return } // Get the index of the segment, and the index of the sector containing // the segment. segmentIndex, err := h.cs.StorageProofSegment(so.id()) if err != nil { h.log.Debugln("Host got an error when fetching a storage proof segment:", err) return } sectorIndex := segmentIndex / (modules.SectorSize / crypto.SegmentSize) // Pull the corresponding sector into memory. sectorRoot := so.SectorRoots[sectorIndex] sectorBytes, err := h.ReadSector(sectorRoot) if err != nil { h.log.Debugln(err) return } // Build the storage proof for just the sector. sectorSegment := segmentIndex % (modules.SectorSize / crypto.SegmentSize) base, cachedHashSet := crypto.MerkleProof(sectorBytes, sectorSegment) // Using the sector, build a cached root. log2SectorSize := uint64(0) for 1<= so.proofDeadline() { h.log.Println("file contract complete, id", so.id()) h.mu.Lock() h.removeStorageObligation(so, obligationSucceeded) h.mu.Unlock() } } Sia-1.0.4/modules/host/storageobligations_smoke_test.go000066400000000000000000000565421302033070500233450ustar00rootroot00000000000000package host // storageobligations_smoke_test.go performs smoke testing on the the storage // obligation management. This includes adding valid storage obligations, and // waiting until they expire, to see if the failure modes are all handled // correctly. import ( "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // randSector creates a random sector, returning the sector along with the // Merkle root of the sector. func randSector() (crypto.Hash, []byte, error) { sectorData, err := crypto.RandBytes(int(modules.SectorSize)) if err != nil { return crypto.Hash{}, nil, err } sectorRoot := crypto.MerkleRoot(sectorData) return sectorRoot, sectorData, nil } // newTesterStorageObligation uses the wallet to create and fund a file // contract that will form the foundation of a storage obligation. func (ht *hostTester) newTesterStorageObligation() (storageObligation, error) { // Create the file contract that will be used in the obligation. builder := ht.wallet.StartTransaction() // Fund the file contract with a payout. The payout needs to be big enough // that the expected revenue is larger than the fee that the host may end // up paying. payout := types.SiacoinPrecision.Mul64(1e3) err := builder.FundSiacoins(payout) if err != nil { return storageObligation{}, err } // Add the file contract that consumes the funds. _ = builder.AddFileContract(types.FileContract{ // Because this file contract needs to be able to accept file contract // revisions, the expiration is put more than // 'revisionSubmissionBuffer' blocks into the future. WindowStart: ht.host.blockHeight + revisionSubmissionBuffer + 2, WindowEnd: ht.host.blockHeight + revisionSubmissionBuffer + defaultWindowSize + 2, Payout: payout, ValidProofOutputs: []types.SiacoinOutput{ { Value: types.PostTax(ht.host.blockHeight, payout), }, { Value: types.ZeroCurrency, }, }, MissedProofOutputs: []types.SiacoinOutput{ { Value: types.PostTax(ht.host.blockHeight, payout), }, { Value: types.ZeroCurrency, }, }, UnlockHash: (types.UnlockConditions{}).UnlockHash(), RevisionNumber: 0, }) // Sign the transaction. tSet, err := builder.Sign(true) if err != nil { return storageObligation{}, err } // Assemble and return the storage obligation. so := storageObligation{ OriginTransactionSet: tSet, // TODO: There are no tracking values, because no fees were added. } return so, nil } // TestBlankStorageObligation checks that the host correctly manages a blank // storage obligation. func TestBlankStorageObligation(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := newHostTester("TestBlankStorageObligation") if err != nil { t.Fatal(err) } defer ht.Close() // The number of contracts reported by the host should be zero. fm := ht.host.FinancialMetrics() if fm.ContractCount != 0 { t.Error("host does not start with 0 contracts:", fm.ContractCount) } // Start by adding a storage obligation to the host. To emulate conditions // of a renter creating the first contract, the storage obligation has no // data, but does have money. so, err := ht.newTesterStorageObligation() if err != nil { t.Fatal(err) } ht.host.managedLockStorageObligation(so.id()) err = ht.host.addStorageObligation(so) if err != nil { t.Fatal(err) } ht.host.managedUnlockStorageObligation(so.id()) // Storage obligation should not be marked as having the transaction // confirmed on the blockchain. if so.OriginConfirmed { t.Fatal("storage obligation should not yet be marked as confirmed, confirmation is on the way") } fm = ht.host.FinancialMetrics() if fm.ContractCount != 1 { t.Error("host should have 1 contract:", fm.ContractCount) } // Mine a block to confirm the transaction containing the storage // obligation. _, err = ht.miner.AddBlock() if err != nil { t.Fatal(err) } err = ht.host.tg.Flush() if err != nil { t.Fatal(err) } // Load the storage obligation from the database, see if it updated // correctly. err = ht.host.db.View(func(tx *bolt.Tx) error { so, err = getStorageObligation(tx, so.id()) if err != nil { return err } return nil }) if err != nil { t.Fatal(err) } if !so.OriginConfirmed { t.Fatal("origin transaction for storage obligation was not confirmed after a block was mined") } // Mine until the host would be submitting a storage proof. Check that the // host has cleared out the storage proof - the consensus code makes it // impossible to submit a storage proof for an empty file contract, so the // host should fail and give up by deleting the storage obligation. for i := types.BlockHeight(0); i <= revisionSubmissionBuffer*2+1; i++ { _, err := ht.miner.AddBlock() if err != nil { t.Fatal(err) } err = ht.host.tg.Flush() if err != nil { t.Fatal(err) } } err = ht.host.db.View(func(tx *bolt.Tx) error { so, err = getStorageObligation(tx, so.id()) if err != nil { return err } return nil }) if err != nil { t.Fatal(err) } fm = ht.host.FinancialMetrics() if fm.ContractCount != 0 { t.Error("host should have 0 contracts, the contracts were all completed:", fm.ContractCount) } } // TestSingleSectorObligationStack checks that the host correctly manages a // storage obligation with a single sector, the revision is created the same // block as the file contract. func TestSingleSectorStorageObligationStack(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := newHostTester("TestSingleSectorStorageObligationStack") if err != nil { t.Fatal(err) } defer ht.Close() // Start by adding a storage obligation to the host. To emulate conditions // of a renter creating the first contract, the storage obligation has no // data, but does have money. so, err := ht.newTesterStorageObligation() if err != nil { t.Fatal(err) } ht.host.managedLockStorageObligation(so.id()) err = ht.host.addStorageObligation(so) if err != nil { t.Fatal(err) } ht.host.managedUnlockStorageObligation(so.id()) // Storage obligation should not be marked as having the transaction // confirmed on the blockchain. if so.OriginConfirmed { t.Fatal("storage obligation should not yet be marked as confirmed, confirmation is on the way") } // Add a file contract revision, moving over a small amount of money to pay // for the file contract. sectorRoot, sectorData, err := randSector() if err != nil { t.Fatal(err) } so.SectorRoots = []crypto.Hash{sectorRoot} sectorCost := types.SiacoinPrecision.Mul64(550) so.PotentialStorageRevenue = so.PotentialStorageRevenue.Add(sectorCost) ht.host.financialMetrics.PotentialStorageRevenue = ht.host.financialMetrics.PotentialStorageRevenue.Add(sectorCost) validPayouts, missedPayouts := so.payouts() validPayouts[0].Value = validPayouts[0].Value.Sub(sectorCost) validPayouts[1].Value = validPayouts[1].Value.Add(sectorCost) missedPayouts[0].Value = missedPayouts[0].Value.Sub(sectorCost) missedPayouts[1].Value = missedPayouts[1].Value.Add(sectorCost) revisionSet := []types.Transaction{{ FileContractRevisions: []types.FileContractRevision{{ ParentID: so.id(), UnlockConditions: types.UnlockConditions{}, NewRevisionNumber: 1, NewFileSize: uint64(len(sectorData)), NewFileMerkleRoot: sectorRoot, NewWindowStart: so.expiration(), NewWindowEnd: so.proofDeadline(), NewValidProofOutputs: validPayouts, NewMissedProofOutputs: missedPayouts, NewUnlockHash: types.UnlockConditions{}.UnlockHash(), }}, }} ht.host.managedLockStorageObligation(so.id()) err = ht.host.modifyStorageObligation(so, nil, []crypto.Hash{sectorRoot}, [][]byte{sectorData}) if err != nil { t.Fatal(err) } ht.host.managedUnlockStorageObligation(so.id()) // Submit the revision set to the transaction pool. err = ht.tpool.AcceptTransactionSet(revisionSet) if err != nil { t.Fatal(err) } // Mine a block to confirm the transactions containing the file contract // and the file contract revision. _, err = ht.miner.AddBlock() if err != nil { t.Fatal(err) } // Load the storage obligation from the database, see if it updated // correctly. err = ht.host.db.View(func(tx *bolt.Tx) error { so, err = getStorageObligation(tx, so.id()) if err != nil { return err } return nil }) if err != nil { t.Fatal(err) } if !so.OriginConfirmed { t.Fatal("origin transaction for storage obligation was not confirmed after a block was mined") } if !so.RevisionConfirmed { t.Fatal("revision transaction for storage obligation was not confirmed after a block was mined") } // Mine until the host submits a storage proof. for i := ht.host.blockHeight; i <= so.expiration()+resubmissionTimeout; i++ { _, err := ht.miner.AddBlock() if err != nil { t.Fatal(err) } } // Flush the host - flush will block until the host has submitted the // storage proof to the transaction pool. err = ht.host.tg.Flush() if err != nil { t.Fatal(err) } // Mine another block, to get the storage proof from the transaction pool // into the blockchain. _, err = ht.miner.AddBlock() if err != nil { t.Fatal(err) } // Grab the storage proof and inspect the contents. err = ht.host.db.View(func(tx *bolt.Tx) error { so, err = getStorageObligation(tx, so.id()) if err != nil { return err } return nil }) if err != nil { t.Fatal(err) } if !so.OriginConfirmed { t.Fatal("origin transaction for storage obligation was not confirmed after a block was mined") } if !so.RevisionConfirmed { t.Fatal("revision transaction for storage obligation was not confirmed after a block was mined") } if !so.ProofConfirmed { t.Fatal("storage obligation is not saying that the storage proof was confirmed on the blockchain") } // Mine blocks until the storage proof has enough confirmations that the // host will finalize the obligation. for i := 0; i <= int(defaultWindowSize); i++ { _, err := ht.miner.AddBlock() if err != nil { t.Fatal(err) } } err = ht.host.db.View(func(tx *bolt.Tx) error { so, err = getStorageObligation(tx, so.id()) if err != nil { return err } if so.SectorRoots != nil { t.Error("sector roots were not cleared when the host finalized the obligation") } if so.ObligationStatus != obligationSucceeded { t.Error("obligation is not being reported as successful:", so.ObligationStatus) } return nil }) if err != nil { t.Fatal(err) } if !ht.host.financialMetrics.StorageRevenue.Equals(sectorCost) { t.Fatal("the host should be reporting revenue after a successful storage proof") } } // TestMultiSectorObligationStack checks that the host correctly manages a // storage obligation with a single sector, the revision is created the same // block as the file contract. // // Unlike the SingleSector test, the multi sector test attempts to spread file // contract revisions over multiple blocks. func TestMultiSectorStorageObligationStack(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := newHostTester("TestMultiSectorStorageObligationStack") if err != nil { t.Fatal(err) } defer ht.Close() // Start by adding a storage obligation to the host. To emulate conditions // of a renter creating the first contract, the storage obligation has no // data, but does have money. so, err := ht.newTesterStorageObligation() if err != nil { t.Fatal(err) } ht.host.managedLockStorageObligation(so.id()) err = ht.host.addStorageObligation(so) if err != nil { t.Fatal(err) } ht.host.managedUnlockStorageObligation(so.id()) // Storage obligation should not be marked as having the transaction // confirmed on the blockchain. if so.OriginConfirmed { t.Fatal("storage obligation should not yet be marked as confirmed, confirmation is on the way") } // Deviation from SingleSector test - mine a block here to confirm the // storage obligation before a file contract revision is created. _, err = ht.miner.AddBlock() if err != nil { t.Fatal(err) } // Load the storage obligation from the database, see if it updated // correctly. err = ht.host.db.View(func(tx *bolt.Tx) error { so, err = getStorageObligation(tx, so.id()) if err != nil { return err } return nil }) if err != nil { t.Fatal(err) } if !so.OriginConfirmed { t.Fatal("origin transaction for storage obligation was not confirmed after a block was mined") } // Add a file contract revision, moving over a small amount of money to pay // for the file contract. sectorRoot, sectorData, err := randSector() if err != nil { t.Fatal(err) } so.SectorRoots = []crypto.Hash{sectorRoot} sectorCost := types.SiacoinPrecision.Mul64(550) so.PotentialStorageRevenue = so.PotentialStorageRevenue.Add(sectorCost) ht.host.financialMetrics.PotentialStorageRevenue = ht.host.financialMetrics.PotentialStorageRevenue.Add(sectorCost) validPayouts, missedPayouts := so.payouts() validPayouts[0].Value = validPayouts[0].Value.Sub(sectorCost) validPayouts[1].Value = validPayouts[1].Value.Add(sectorCost) missedPayouts[0].Value = missedPayouts[0].Value.Sub(sectorCost) missedPayouts[1].Value = missedPayouts[1].Value.Add(sectorCost) revisionSet := []types.Transaction{{ FileContractRevisions: []types.FileContractRevision{{ ParentID: so.id(), UnlockConditions: types.UnlockConditions{}, NewRevisionNumber: 1, NewFileSize: uint64(len(sectorData)), NewFileMerkleRoot: sectorRoot, NewWindowStart: so.expiration(), NewWindowEnd: so.proofDeadline(), NewValidProofOutputs: validPayouts, NewMissedProofOutputs: missedPayouts, NewUnlockHash: types.UnlockConditions{}.UnlockHash(), }}, }} ht.host.managedLockStorageObligation(so.id()) err = ht.host.modifyStorageObligation(so, nil, []crypto.Hash{sectorRoot}, [][]byte{sectorData}) if err != nil { t.Fatal(err) } ht.host.managedUnlockStorageObligation(so.id()) // Submit the revision set to the transaction pool. err = ht.tpool.AcceptTransactionSet(revisionSet) if err != nil { t.Fatal(err) } // Create a second file contract revision, which is going to be submitted // to the transaction pool after the first revision. Though, in practice // this should never happen, we want to check that the transaction pool is // correctly handling multiple file contract revisions being submitted in // the same block cycle. This test will additionally tell us whether or not // the host can correctly handle building storage proofs for files with // multiple sectors. sectorRoot2, sectorData2, err := randSector() if err != nil { t.Fatal(err) } so.SectorRoots = []crypto.Hash{sectorRoot, sectorRoot2} sectorCost2 := types.SiacoinPrecision.Mul64(650) so.PotentialStorageRevenue = so.PotentialStorageRevenue.Add(sectorCost2) ht.host.financialMetrics.PotentialStorageRevenue = ht.host.financialMetrics.PotentialStorageRevenue.Add(sectorCost2) validPayouts, missedPayouts = so.payouts() validPayouts[0].Value = validPayouts[0].Value.Sub(sectorCost2) validPayouts[1].Value = validPayouts[1].Value.Add(sectorCost2) missedPayouts[0].Value = missedPayouts[0].Value.Sub(sectorCost2) missedPayouts[1].Value = missedPayouts[1].Value.Add(sectorCost2) combinedSectors := append(sectorData, sectorData2...) combinedRoot := crypto.MerkleRoot(combinedSectors) revisionSet2 := []types.Transaction{{ FileContractRevisions: []types.FileContractRevision{{ ParentID: so.id(), UnlockConditions: types.UnlockConditions{}, NewRevisionNumber: 2, NewFileSize: uint64(len(sectorData) + len(sectorData2)), NewFileMerkleRoot: combinedRoot, NewWindowStart: so.expiration(), NewWindowEnd: so.proofDeadline(), NewValidProofOutputs: validPayouts, NewMissedProofOutputs: missedPayouts, NewUnlockHash: types.UnlockConditions{}.UnlockHash(), }}, }} ht.host.managedLockStorageObligation(so.id()) err = ht.host.modifyStorageObligation(so, nil, []crypto.Hash{sectorRoot2}, [][]byte{sectorData2}) if err != nil { t.Fatal(err) } ht.host.managedUnlockStorageObligation(so.id()) // Submit the revision set to the transaction pool. err = ht.tpool.AcceptTransactionSet(revisionSet2) if err != nil { t.Fatal(err) } // Mine a block to confirm the transactions containing the file contract // and the file contract revision. _, err = ht.miner.AddBlock() if err != nil { t.Fatal(err) } // Load the storage obligation from the database, see if it updated // correctly. err = ht.host.db.View(func(tx *bolt.Tx) error { so, err = getStorageObligation(tx, so.id()) if err != nil { return err } return nil }) if err != nil { t.Fatal(err) } if !so.OriginConfirmed { t.Fatal("origin transaction for storage obligation was not confirmed after a block was mined") } if !so.RevisionConfirmed { t.Fatal("revision transaction for storage obligation was not confirmed after a block was mined") } // Mine until the host submits a storage proof. for i := ht.host.blockHeight; i <= so.expiration()+resubmissionTimeout; i++ { _, err := ht.miner.AddBlock() if err != nil { t.Fatal(err) } } // Flush the host - flush will block until the host has submitted the // storage proof to the transaction pool. err = ht.host.tg.Flush() if err != nil { t.Fatal(err) } // Mine another block, to get the storage proof from the transaction pool // into the blockchain. _, err = ht.miner.AddBlock() if err != nil { t.Fatal(err) } err = ht.host.db.View(func(tx *bolt.Tx) error { so, err = getStorageObligation(tx, so.id()) if err != nil { return err } return nil }) if err != nil { t.Fatal(err) } if !so.OriginConfirmed { t.Fatal("origin transaction for storage obligation was not confirmed after a block was mined") } if !so.RevisionConfirmed { t.Fatal("revision transaction for storage obligation was not confirmed after a block was mined") } if !so.ProofConfirmed { t.Fatal("storage obligation is not saying that the storage proof was confirmed on the blockchain") } // Mine blocks until the storage proof has enough confirmations that the // host will delete the file entirely. for i := 0; i <= int(defaultWindowSize); i++ { _, err := ht.miner.AddBlock() if err != nil { t.Fatal(err) } } err = ht.host.db.View(func(tx *bolt.Tx) error { so, err = getStorageObligation(tx, so.id()) if err != nil { return err } if so.SectorRoots != nil { t.Error("sector roots were not cleared out when the storage proof was finalized") } if so.ObligationStatus != obligationSucceeded { t.Error("storage obligation was not reported as a success") } return nil }) if err != nil { t.Fatal(err) } if !ht.host.financialMetrics.StorageRevenue.Equals(sectorCost.Add(sectorCost2)) { t.Fatal("the host should be reporting revenue after a successful storage proof") } } // TestAutoRevisionSubmission checks that the host correctly submits a file // contract revision to the consensus set. func TestAutoRevisionSubmission(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := newHostTester("TestAutoRevisionSubmission") if err != nil { t.Fatal(err) } defer ht.Close() // Start by adding a storage obligation to the host. To emulate conditions // of a renter creating the first contract, the storage obligation has no // data, but does have money. so, err := ht.newTesterStorageObligation() if err != nil { t.Fatal(err) } ht.host.managedLockStorageObligation(so.id()) err = ht.host.addStorageObligation(so) if err != nil { t.Fatal(err) } ht.host.managedUnlockStorageObligation(so.id()) // Storage obligation should not be marked as having the transaction // confirmed on the blockchain. if so.OriginConfirmed { t.Fatal("storage obligation should not yet be marked as confirmed, confirmation is on the way") } // Add a file contract revision, moving over a small amount of money to pay // for the file contract. sectorRoot, sectorData, err := randSector() if err != nil { t.Fatal(err) } so.SectorRoots = []crypto.Hash{sectorRoot} sectorCost := types.SiacoinPrecision.Mul64(550) so.PotentialStorageRevenue = so.PotentialStorageRevenue.Add(sectorCost) ht.host.financialMetrics.PotentialStorageRevenue = ht.host.financialMetrics.PotentialStorageRevenue.Add(sectorCost) validPayouts, missedPayouts := so.payouts() validPayouts[0].Value = validPayouts[0].Value.Sub(sectorCost) validPayouts[1].Value = validPayouts[1].Value.Add(sectorCost) missedPayouts[0].Value = missedPayouts[0].Value.Sub(sectorCost) missedPayouts[1].Value = missedPayouts[1].Value.Add(sectorCost) revisionSet := []types.Transaction{{ FileContractRevisions: []types.FileContractRevision{{ ParentID: so.id(), UnlockConditions: types.UnlockConditions{}, NewRevisionNumber: 1, NewFileSize: uint64(len(sectorData)), NewFileMerkleRoot: sectorRoot, NewWindowStart: so.expiration(), NewWindowEnd: so.proofDeadline(), NewValidProofOutputs: validPayouts, NewMissedProofOutputs: missedPayouts, NewUnlockHash: types.UnlockConditions{}.UnlockHash(), }}, }} so.RevisionTransactionSet = revisionSet ht.host.managedLockStorageObligation(so.id()) err = ht.host.modifyStorageObligation(so, nil, []crypto.Hash{sectorRoot}, [][]byte{sectorData}) if err != nil { t.Fatal(err) } ht.host.managedUnlockStorageObligation(so.id()) err = ht.host.tg.Flush() if err != nil { t.Fatal(err) } // Unlike the other tests, this test does not submit the file contract // revision to the transaction pool for the host, the host is expected to // do it automatically. // Mine until the host submits a storage proof. for i := types.BlockHeight(0); i <= revisionSubmissionBuffer+2+resubmissionTimeout; i++ { _, err := ht.miner.AddBlock() if err != nil { t.Fatal(err) } err = ht.host.tg.Flush() if err != nil { t.Fatal(err) } } // Flush the host - flush will block until the host has submitted the // storage proof to the transaction pool. err = ht.host.tg.Flush() if err != nil { t.Fatal(err) } // Mine another block, to get the storage proof from the transaction pool // into the blockchain. _, err = ht.miner.AddBlock() if err != nil { t.Fatal(err) } err = ht.host.db.View(func(tx *bolt.Tx) error { so, err = getStorageObligation(tx, so.id()) if err != nil { return err } return nil }) if err != nil { t.Fatal(err) } if !so.OriginConfirmed { t.Fatal("origin transaction for storage obligation was not confirmed after blocks were mined") } if !so.RevisionConfirmed { t.Fatal("revision transaction for storage obligation was not confirmed after blocks were mined") } if !so.ProofConfirmed { t.Fatal("storage obligation is not saying that the storage proof was confirmed on the blockchain") } // Mine blocks until the storage proof has enough confirmations that the // host will delete the file entirely. for i := 0; i <= int(defaultWindowSize); i++ { _, err := ht.miner.AddBlock() if err != nil { t.Fatal(err) } err = ht.host.tg.Flush() if err != nil { t.Fatal(err) } } err = ht.host.db.View(func(tx *bolt.Tx) error { so, err = getStorageObligation(tx, so.id()) if err != nil { return err } if so.SectorRoots != nil { t.Error("sector roots were not cleared out when the storage proof was finalized") } if so.ObligationStatus != obligationSucceeded { t.Error("storage obligation was not reported as a success") } return nil }) if err != nil { t.Fatal(err) } if !ht.host.financialMetrics.StorageRevenue.Equals(sectorCost) { t.Fatal("the host should be reporting revenue after a successful storage proof") } } Sia-1.0.4/modules/host/storageobligations_test.go000066400000000000000000000042061302033070500221350ustar00rootroot00000000000000package host import ( "testing" "github.com/NebulousLabs/Sia/types" ) // TestStorageObligationID checks that the return function of the storage // obligation returns the correct value for the obligaiton id. func TestStorageObligationID(t *testing.T) { t.Parallel() // Try a transaction set with just a file contract. so1 := &storageObligation{ OriginTransactionSet: []types.Transaction{{ FileContracts: []types.FileContract{{ ValidProofOutputs: []types.SiacoinOutput{ { UnlockHash: types.UnlockHash{2, 1, 3}, Value: types.NewCurrency64(35), }, { UnlockHash: types.UnlockHash{0, 1, 3}, Value: types.NewCurrency64(25), }, }, MissedProofOutputs: []types.SiacoinOutput{ { UnlockHash: types.UnlockHash{110, 1, 3}, Value: types.NewCurrency64(3325), }, { UnlockHash: types.UnlockHash{110, 1, 3}, Value: types.NewCurrency64(8325), }, }, }}, }}, } if so1.id() != so1.OriginTransactionSet[0].FileContractID(0) { t.Error("id function of storage obligation is not correct") } // Try a file contract that includes file contract dependencies. so2 := &storageObligation{ OriginTransactionSet: []types.Transaction{ { SiacoinOutputs: []types.SiacoinOutput{{ UnlockHash: types.UnlockHash{1, 3, 2}, Value: types.NewCurrency64(5), }}, }, { FileContracts: []types.FileContract{{ ValidProofOutputs: []types.SiacoinOutput{ { UnlockHash: types.UnlockHash{8, 11, 4}, Value: types.NewCurrency64(85), }, { UnlockHash: types.UnlockHash{8, 11, 14}, Value: types.NewCurrency64(859), }, }, MissedProofOutputs: []types.SiacoinOutput{ { UnlockHash: types.UnlockHash{8, 113, 4}, Value: types.NewCurrency64(853), }, { UnlockHash: types.UnlockHash{8, 119, 14}, Value: types.NewCurrency64(9859), }, }, }}, }, }, } if so2.id() != so2.OriginTransactionSet[1].FileContractID(0) { t.Error("id function of storage obligation incorrect for file contracts with dependencies") } } Sia-1.0.4/modules/host/storageobligationslock.go000066400000000000000000000042611302033070500217500ustar00rootroot00000000000000package host import ( "errors" "github.com/NebulousLabs/Sia/sync" "github.com/NebulousLabs/Sia/types" ) var ( // errObligationLocked is returned if the file contract being requested is // currently locked. The lock can be in place if there is a storage proof // being submitted, if there is another renter altering the contract, or if // there have been network connections with have not resolved yet. errObligationLocked = errors.New("the requested file contract is currently locked") ) // managedLockStorageObligation puts a storage obligation under lock in the // host. func (h *Host) managedLockStorageObligation(soid types.FileContractID) { // Check if a lock has been created for this storage obligation. If not, // create one. The map must be accessed under lock, but the request for the // storage lock must not be made under lock. h.mu.Lock() tl, exists := h.lockedStorageObligations[soid] if !exists { tl = new(sync.TryMutex) h.lockedStorageObligations[soid] = tl } h.mu.Unlock() tl.Lock() } // managedTryLockStorageObligation attempts to put a storage obligation under // lock, returning an error if the lock cannot be obtained. func (h *Host) managedTryLockStorageObligation(soid types.FileContractID) error { // Check if a lock has been created for this storage obligation. If not, // create one. The map must be accessed under lock, but the request for the // storage lock must not be made under lock. h.mu.Lock() tl, exists := h.lockedStorageObligations[soid] if !exists { tl = new(sync.TryMutex) h.lockedStorageObligations[soid] = tl } h.mu.Unlock() if tl.TryLockTimed(obligationLockTimeout) { return nil } return errObligationLocked } // managedUnlockStorageObligation takes a storage obligation out from under lock in // the host. func (h *Host) managedUnlockStorageObligation(soid types.FileContractID) { // Check if a lock has been created for this storage obligation. If not, // create one. The map must be accessed under lock, but the request for the // storage lock must not be made under lock. h.mu.Lock() tl, exists := h.lockedStorageObligations[soid] if !exists { h.log.Critical(errObligationUnlocked) return } h.mu.Unlock() tl.Unlock() } Sia-1.0.4/modules/host/storageobligationslock_test.go000066400000000000000000000053471302033070500230150ustar00rootroot00000000000000package host import ( "testing" "time" "github.com/NebulousLabs/Sia/types" ) // TestObligationLocks checks that the storage obligation locking functions // properly blocks and errors out for various use cases. func TestObligationLocks(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := blankHostTester("TestObligationLocks") if err != nil { t.Fatal(err) } defer ht.Close() // Simple lock and unlock. ob1 := types.FileContractID{1} ht.host.managedLockStorageObligation(ob1) ht.host.managedUnlockStorageObligation(ob1) // Simple lock and unlock, with trylock. err = ht.host.managedTryLockStorageObligation(ob1) if err != nil { t.Fatal("unable to get lock despite not having a lock in place") } ht.host.managedUnlockStorageObligation(ob1) // Threaded lock and unlock. blockSuccessful := false ht.host.managedLockStorageObligation(ob1) go func() { time.Sleep(obligationLockTimeout * 2) blockSuccessful = true ht.host.managedUnlockStorageObligation(ob1) }() ht.host.managedLockStorageObligation(ob1) if !blockSuccessful { t.Error("two threads were able to simultaneously grab an obligation lock") } ht.host.managedUnlockStorageObligation(ob1) // Attempted lock and unlock - failed. ht.host.managedLockStorageObligation(ob1) go func() { time.Sleep(obligationLockTimeout * 2) ht.host.managedUnlockStorageObligation(ob1) }() err = ht.host.managedTryLockStorageObligation(ob1) if err != errObligationLocked { t.Fatal("storage obligation was able to get a lock, despite already being locked") } // Attempted lock and unlock - succeeded. ht.host.managedLockStorageObligation(ob1) go func() { time.Sleep(obligationLockTimeout / 2) ht.host.managedUnlockStorageObligation(ob1) }() err = ht.host.managedTryLockStorageObligation(ob1) if err != nil { t.Fatal("storage obligation unable to get lock, depsite having enough time") } ht.host.managedUnlockStorageObligation(ob1) // Multiple locks and unlocks happening together. ob2 := types.FileContractID{2} ob3 := types.FileContractID{3} ht.host.managedLockStorageObligation(ob1) ht.host.managedLockStorageObligation(ob2) ht.host.managedLockStorageObligation(ob3) ht.host.managedUnlockStorageObligation(ob3) ht.host.managedUnlockStorageObligation(ob2) err = ht.host.managedTryLockStorageObligation(ob2) if err != nil { t.Fatal("unable to get lock despite not having a lock in place") } err = ht.host.managedTryLockStorageObligation(ob3) if err != nil { t.Fatal("unable to get lock despite not having a lock in place") } err = ht.host.managedTryLockStorageObligation(ob1) if err != errObligationLocked { t.Fatal("storage obligation was able to get a lock, despite already being locked") } ht.host.managedUnlockStorageObligation(ob1) } Sia-1.0.4/modules/host/testdata/000077500000000000000000000000001302033070500164575ustar00rootroot00000000000000Sia-1.0.4/modules/host/testdata/persist_compat_1.0.0.json000066400000000000000000000047471302033070500231360ustar00rootroot00000000000000"Sia Host" "0.5" { "DownloadCalls": 123, "ErroredCalls": 79, "FormContractCalls": 38, "RenewCalls": 3520, "ReviseCalls": 93, "RecentRevisionCalls": 6220, "SettingsCalls": 31096, "UnrecognizedCalls": 56, "BlockHeight": 55993, "RecentChange": [ 175, 108, 152, 227, 113, 126, 172, 236, 162, 126, 128, 253, 139, 101, 211, 17, 166, 161, 56, 255, 58, 119, 62, 129, 52, 163, 171, 193, 51, 17, 131, 151 ], "Announced": true, "AutoAddress": "[2607:5300:60:e48::1]:9982", "FinancialMetrics": { "contractcompensation": "530", "potentialcontractcompensation": "660000000000000000000000000", "lockedstoragecollateral": "305845953422220264808120320", "lostrevenue": "310", "loststoragecollateral": "5620", "potentialerevenue": "5743380859257237589196800", "riskedstoragecollateral": "14358452148148056254054400", "storagerevenue": "6243", "transactionfeeexpenses": "12", "downloadbandwidthrevenue": "64", "potentialdownloadbandwidthrevenue": "1048576000000000000000000", "potentialuploadbandwidthrevenue": "205520896000000000000000", "uploadbandwidthrevenue": "694" }, "PublicKey": { "algorithm": "ed25519", "key": "6ZcZSi+y/GUfHzpCdHJW4aryq/d2wOEUmCZFcbRovdo=" }, "RevisionNumber": 31459, "SecretKey": [ 228, 68, 158, 12, 168, 20, 77, 161, 132, 36, 71, 178, 130, 41, 135, 233, 241, 61, 233, 4, 85, 41, 115, 137, 243, 210, 225, 74, 50, 184, 116, 106, 233, 151, 25, 74, 47, 178, 252, 101, 31, 31, 58, 66, 116, 114, 86, 225, 170, 242, 171, 247, 118, 192, 225, 20, 152, 38, 69, 113, 180, 104, 189, 218 ], "Settings": { "acceptingcontracts": true, "maxduration": 25920, "maxdownloadbatchsize": 17825792, "maxrevisebatchsize": 17825792, "netaddress": "fakeaddr.com:5310", "windowsize": 144, "collateral": "5787037037037", "collateralbudget": "5000000000000000000000000000000", "maxcollateral": "250000000000000000000000000000", "downloadlimitgrowth": 646410, "downloadlimitcap": 6410, "downloadspeedlimit": 96960, "uploadlimitgrowth": 240, "uploadlimitcap": 640, "uploadspeedlimit": 4820, "contractprice": "30000000000000000000000000", "minimumdownloadbandwidthprice": "10000000000000000", "storageprice": "2314814814814", "minimumuploadbandwidthprice": "1000000000000000" }, "UnlockHash": "593a0322c9dd3d3c0624689e2ee4849cbf0008f2d25dac10c84ae3e3124b712e333deb252ad3" } Sia-1.0.4/modules/host/update.go000066400000000000000000000243061302033070500164640ustar00rootroot00000000000000package host // TODO: Need to check that 'RevisionConfirmed' is sensitive to whether or not // it was the *most recent* revision that got confirmed. import ( "encoding/binary" "encoding/json" "sync" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // initRescan is a helper function of initConsensusSubscribe, and is called when // the host and the consensus set have become desynchronized. Desynchronization // typically happens if the user is replacing or altering the persistent files // in the consensus set or the host. func (h *Host) initRescan() error { // Reset all of the variables that have relevance to the consensus set. var allObligations []storageObligation // Reset all of the consensus-relevant variables in the host. h.blockHeight = 0 // Reset all of the storage obligations. err := h.db.Update(func(tx *bolt.Tx) error { bsu := tx.Bucket(bucketStorageObligations) c := bsu.Cursor() for k, soBytes := c.First(); soBytes != nil; k, soBytes = c.Next() { var so storageObligation err := json.Unmarshal(soBytes, &so) if err != nil { return err } so.OriginConfirmed = false so.RevisionConfirmed = false so.ProofConfirmed = false allObligations = append(allObligations, so) soBytes, err = json.Marshal(so) if err != nil { return err } err = bsu.Put(k, soBytes) if err != nil { return err } } return nil }) if err != nil { return err } // Subscribe to the consensus set. This is a blocking call that will not // return until the host has fully caught up to the current block. // // Convention dictates that the host should not make external calls while // under lock, but this function happens at startup while blocking. Because // it happens while blocking, and because there is no actual host lock held // at this time, none of the host external functions are exposed, so it is // save to make the exported call. err = h.cs.ConsensusSetSubscribe(h, modules.ConsensusChangeBeginning) if err != nil { return err } h.tg.OnStop(func() { h.cs.Unsubscribe(h) }) // Re-queue all of the action items for the storage obligations. for _, so := range allObligations { soid := so.id() err0 := h.tpool.AcceptTransactionSet(so.OriginTransactionSet) err1 := h.queueActionItem(h.blockHeight+resubmissionTimeout, soid) err2 := h.queueActionItem(so.expiration()-revisionSubmissionBuffer, soid) err3 := h.queueActionItem(so.expiration()+resubmissionTimeout, soid) err = composeErrors(err0, err1, err2, err3) if err != nil { h.log.Println("dropping storage obligation during rescan, id", so.id()) return composeErrors(err, h.removeStorageObligation(so, obligationRejected)) } } return nil } // initConsensusSubscription subscribes the host to the consensus set. func (h *Host) initConsensusSubscription() error { // Convention dictates that the host should not make external calls while // under lock, but this function happens at startup while blocking. Because // it happens while blocking, and because there is no actual host lock held // at this time, none of the host external functions are exposed, so it is // save to make the exported call. err := h.cs.ConsensusSetSubscribe(h, h.recentChange) if err == modules.ErrInvalidConsensusChangeID { // Perform a rescan of the consensus set if the change id that the host // has is unrecognized by the consensus set. This will typically only // happen if the user has been replacing files inside the Sia folder // structure. return h.initRescan() } if err != nil { return err } h.tg.OnStop(func() { h.cs.Unsubscribe(h) }) return nil } // ProcessConsensusChange will be called by the consensus set every time there // is a change to the blockchain. func (h *Host) ProcessConsensusChange(cc modules.ConsensusChange) { // Add is called at the beginning of the function, but Done cannot be // called until all of the threads spawned by this function have also // terminated. This function should not block while these threads wait to // terminate. h.mu.Lock() defer h.mu.Unlock() err := h.tg.Add() if err != nil { return } wg := new(sync.WaitGroup) defer func() { go func() { wg.Wait() h.tg.Done() }() }() // Wrap the whole parsing into a single large database tx to keep things // efficient. var actionItems []types.FileContractID err = h.db.Update(func(tx *bolt.Tx) error { for _, block := range cc.RevertedBlocks { // Look for transactions relevant to open storage obligations. for _, txn := range block.Transactions { // Check for file contracts. if len(txn.FileContracts) > 0 { for j := range txn.FileContracts { fcid := txn.FileContractID(uint64(j)) so, err := getStorageObligation(tx, fcid) if err != nil { // The storage folder may not exist, or the disk // may be having trouble. Either way, we ignore the // problem. If the disk is having trouble, the user // will have to perform a rescan. continue } so.OriginConfirmed = false err = putStorageObligation(tx, so) if err != nil { continue } } } // Check for file contract revisions. if len(txn.FileContractRevisions) > 0 { for _, fcr := range txn.FileContractRevisions { so, err := getStorageObligation(tx, fcr.ParentID) if err != nil { // The storage folder may not exist, or the disk // may be having trouble. Either way, we ignore the // problem. If the disk is having trouble, the user // will have to perform a rescan. continue } so.RevisionConfirmed = false err = putStorageObligation(tx, so) if err != nil { continue } } } // Check for storage proofs. if len(txn.StorageProofs) > 0 { for _, sp := range txn.StorageProofs { // Check database for relevant storage proofs. so, err := getStorageObligation(tx, sp.ParentID) if err != nil { // The storage folder may not exist, or the disk // may be having trouble. Either way, we ignore the // problem. If the disk is having trouble, the user // will have to perform a rescan. continue } so.ProofConfirmed = false err = putStorageObligation(tx, so) if err != nil { continue } } } } // Height is not adjusted when dealing with the genesis block because // the default height is 0 and the genesis block height is 0. If // removing the genesis block, height will already be at height 0 and // should not update, lest an underflow occur. if block.ID() != types.GenesisID { h.blockHeight-- } } for _, block := range cc.AppliedBlocks { // Look for transactions relevant to open storage obligations. for _, txn := range block.Transactions { // Check for file contracts. if len(txn.FileContracts) > 0 { for i := range txn.FileContracts { fcid := txn.FileContractID(uint64(i)) so, err := getStorageObligation(tx, fcid) if err != nil { // The storage folder may not exist, or the disk // may be having trouble. Either way, we ignore the // problem. If the disk is having trouble, the user // will have to perform a rescan. continue } so.OriginConfirmed = true err = putStorageObligation(tx, so) if err != nil { continue } } } // Check for file contract revisions. if len(txn.FileContractRevisions) > 0 { for _, fcr := range txn.FileContractRevisions { so, err := getStorageObligation(tx, fcr.ParentID) if err != nil { // The storage folder may not exist, or the disk // may be having trouble. Either way, we ignore the // problem. If the disk is having trouble, the user // will have to perform a rescan. continue } so.RevisionConfirmed = true err = putStorageObligation(tx, so) if err != nil { continue } } } // Check for storage proofs. if len(txn.StorageProofs) > 0 { for _, sp := range txn.StorageProofs { so, err := getStorageObligation(tx, sp.ParentID) if err != nil { // The storage folder may not exist, or the disk // may be having trouble. Either way, we ignore the // problem. If the disk is having trouble, the user // will have to perform a rescan. continue } so.ProofConfirmed = true err = putStorageObligation(tx, so) if err != nil { continue } } } } // Height is not adjusted when dealing with the genesis block because // the default height is 0 and the genesis block height is 0. If adding // the genesis block, height will already be at height 0 and should not // update. if block.ID() != types.GenesisID { h.blockHeight++ } // Handle any action items relevant to the current height. bai := tx.Bucket(bucketActionItems) heightBytes := make([]byte, 8) binary.BigEndian.PutUint64(heightBytes, uint64(h.blockHeight)) // BigEndian used so bolt will keep things sorted automatically. existingItems := bai.Get(heightBytes) // From the existing items, pull out a storage obligation. knownActionItems := make(map[types.FileContractID]struct{}) obligationIDs := make([]types.FileContractID, len(existingItems)/crypto.HashSize) for i := 0; i < len(existingItems); i += crypto.HashSize { copy(obligationIDs[i/crypto.HashSize][:], existingItems[i:i+crypto.HashSize]) } for _, soid := range obligationIDs { _, exists := knownActionItems[soid] if !exists { actionItems = append(actionItems, soid) knownActionItems[soid] = struct{}{} } } } return nil }) if err != nil { h.log.Println(err) } for i := range actionItems { // Add the action item to the wait group outside of the threaded call. // The call to wg.Done() was established at the beginning of the // function in a defer statement. wg.Add(1) go h.threadedHandleActionItem(actionItems[i], wg) } // Update the host's recent change pointer to point to the most recent // change. h.recentChange = cc.ID // Save the host. err = h.save() if err != nil { h.log.Println("ERROR: could not save during ProcessConsensusChange:", err) } } Sia-1.0.4/modules/host/update_test.go000066400000000000000000000117261302033070500175250ustar00rootroot00000000000000package host import ( "crypto/rand" "io/ioutil" "path/filepath" "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // TestStorageProof checks that the host can create and submit a storage proof. func TestStorageProof(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := newHostTester("TestStorageProof") if err != nil { t.Fatal(err) } defer ht.Close() // create a file contract fc := types.FileContract{ WindowStart: types.MaturityDelay + 3, WindowEnd: 1000, Payout: types.NewCurrency64(1), UnlockHash: types.UnlockConditions{}.UnlockHash(), ValidProofOutputs: []types.SiacoinOutput{{Value: types.NewCurrency64(1)}, {Value: types.NewCurrency64(0)}}, MissedProofOutputs: []types.SiacoinOutput{{Value: types.NewCurrency64(1)}, {Value: types.NewCurrency64(0)}}, } txnBuilder := ht.wallet.StartTransaction() err = txnBuilder.FundSiacoins(fc.Payout) if err != nil { t.Fatal(err) } txnBuilder.AddFileContract(fc) signedTxnSet, err := txnBuilder.Sign(true) if err != nil { t.Fatal(err) } fcid := signedTxnSet[len(signedTxnSet)-1].FileContractID(0) // generate data const dataSize = 777 data := make([]byte, dataSize) _, err = rand.Read(data) if err != nil { t.Fatal(err) } root := crypto.MerkleRoot(data) err = ioutil.WriteFile(filepath.Join(ht.host.persistDir, "foo"), data, 0777) if err != nil { t.Fatal(err) } // create revision rev := types.FileContractRevision{ ParentID: fcid, UnlockConditions: types.UnlockConditions{}, NewFileSize: dataSize, NewWindowStart: fc.WindowStart, NewFileMerkleRoot: root, NewWindowEnd: fc.WindowEnd, NewValidProofOutputs: fc.ValidProofOutputs, NewMissedProofOutputs: fc.MissedProofOutputs, NewRevisionNumber: 1, } _ = types.Transaction{ FileContractRevisions: []types.FileContractRevision{rev}, } /* // create obligation obligation := &contractObligation{ ID: fcid, OriginTransaction: types.Transaction{ FileContracts: []types.FileContract{fc}, }, Path: filepath.Join(ht.host.persistDir, "foo"), } ht.host.obligationsByID[fcid] = obligation ht.host.addActionItem(fc.WindowStart+1, obligation) // submit both to tpool err = ht.tpool.AcceptTransactionSet(append(signedTxnSet, revTxn)) if err != nil { t.Fatal(err) } _, err = ht.miner.AddBlock() if err != nil { t.Fatal(err) } // storage proof will be submitted after mining one more block _, err = ht.miner.AddBlock() if err != nil { t.Fatal(err) } */ } // TestInitRescan probes the initRescan function, verifying that it works in // the naive case. The rescan is triggered manually. func TestInitRescan(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := newHostTester("TestInitRescan") if err != nil { t.Fatal(err) } defer ht.Close() // Check that the host's persistent variables have incorporated the first // few blocks. if ht.host.recentChange == (modules.ConsensusChangeID{}) || ht.host.blockHeight == 0 { t.Fatal("host variables do not indicate that the host is tracking the consensus set correctly") } oldChange := ht.host.recentChange oldHeight := ht.host.blockHeight // Corrupt the variables and perform a rescan to see if they reset // correctly. ht.host.recentChange[0]++ ht.host.blockHeight += 100e3 ht.cs.Unsubscribe(ht.host) err = ht.host.initRescan() if err != nil { t.Fatal(err) } if oldChange != ht.host.recentChange || oldHeight != ht.host.blockHeight { t.Error("consensus tracking variables were not reset correctly after rescan") } } // TestIntegrationAutoRescan checks that a rescan is triggered during New if // the consensus set becomes desynchronized. func TestIntegrationAutoRescan(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ht, err := newHostTester("TestIntegrationAutoRescan") if err != nil { t.Fatal(err) } defer ht.Close() // Check that the host's persistent variables have incorporated the first // few blocks. if ht.host.recentChange == (modules.ConsensusChangeID{}) || ht.host.blockHeight == 0 { t.Fatal("host variables do not indicate that the host is tracking the consensus set correctly") } oldChange := ht.host.recentChange oldHeight := ht.host.blockHeight // Corrupt the variables, then close the host. ht.host.recentChange[0]++ ht.host.blockHeight += 100e3 err = ht.host.Close() // host saves upon closing if err != nil { t.Fatal(err) } // Create a new host and check that the persist variables have correctly // reset. h, err := New(ht.cs, ht.tpool, ht.wallet, "localhost:0", filepath.Join(ht.persistDir, modules.HostDir)) if err != nil { t.Fatal(err) } if oldChange != h.recentChange || oldHeight != h.blockHeight { t.Error("consensus tracking variables were not reset correctly after rescan") } // Set ht.host to 'h' so that the 'ht.Close()' method will close everything // cleanly. ht.host = h } Sia-1.0.4/modules/host/upnp.go000066400000000000000000000112061302033070500161570ustar00rootroot00000000000000package host import ( "errors" "io" "io/ioutil" "net" "net/http" "strconv" "strings" "time" "github.com/NebulousLabs/go-upnp" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" ) // managedLearnHostname discovers the external IP of the Host. If the host's // net address is blank and the host's auto address appears to have changed, // the host will make an announcement on the blockchain. func (h *Host) managedLearnHostname() { if build.Release == "testing" { return } h.mu.RLock() netAddr := h.settings.NetAddress h.mu.RUnlock() // If the settings indicate that an address has been manually set, there is // no reason to learn the hostname. if netAddr != "" { return } // try UPnP first, then fallback to myexternalip.com var hostname string d, err := upnp.Discover() if err == nil { hostname, err = d.ExternalIP() } if err != nil { hostname, err = myExternalIP() } if err != nil { h.log.Println("WARN: failed to discover external IP") return } h.mu.Lock() defer h.mu.Unlock() autoAddress := modules.NetAddress(net.JoinHostPort(hostname, h.port)) if err := autoAddress.IsValid(); err != nil { h.log.Printf("WARN: discovered hostname %q is invalid: %v", autoAddress, err) return } if autoAddress == h.autoAddress && h.announced { // Nothing to do - the auto address has not changed and the previous // annoucement was successful. return } h.autoAddress = autoAddress err = h.save() if err != nil { h.log.Println(err) } // Announce the host, but only if the host is either accepting contracts or // has a storage obligation. If the host is not accepting contracts and has // no open contracts, there is no reason to notify anyone that the host's // address has changed. if h.settings.AcceptingContracts || h.financialMetrics.ContractCount > 0 { err = h.announce(autoAddress) if err != nil { // Set h.announced to false, as the address has changed yet the // renewed annoucement has failed. h.announced = false h.log.Debugln("unable to announce address after upnp-detected address change:", err) } } } // managedForwardPort adds a port mapping to the router. func (h *Host) managedForwardPort(port string) error { if build.Release == "testing" { // Add a blocking placeholder where testing is able to mock behaviors // such as a port forward action that blocks for 10 seconds before // completing. if h.dependencies.disrupt("managedForwardPort") { return nil } // Port forwarding functions are frequently unavailable during testing, // and the long blocking can be highly disruptive. Under normal // scenarios, return without complaint, and without running the // port-forward logic. return nil } // If the port is invalid, there is no need to perform any of the other // tasks. portInt, err := strconv.Atoi(port) if err != nil { return err } d, err := upnp.Discover() if err != nil { h.log.Printf("WARN: could not automatically forward port %s: %v", port, err) return err } err = d.Forward(uint16(portInt), "Sia Host") if err != nil { h.log.Printf("WARN: could not automatically forward port %s: %v", port, err) return err } h.log.Println("INFO: successfully forwarded port", port) return nil } // managedClearPort removes a port mapping from the router. func (h *Host) managedClearPort() error { if build.Release == "testing" { // Allow testing to force an error to be returned here. if h.dependencies.disrupt("managedClearPort return error") { return errors.New("Mocked managedClearPortErr") } return nil } // If the port is invalid, there is no need to perform any of the other // tasks. h.mu.RLock() port := h.port h.mu.RUnlock() portInt, err := strconv.Atoi(port) if err != nil { return err } d, err := upnp.Discover() if err != nil { return err } err = d.Clear(uint16(portInt)) if err != nil { return err } h.log.Println("INFO: successfully unforwarded port", port) return nil } // myExternalIP discovers the host's external IP by querying a centralized // service, http://myexternalip.com. func myExternalIP() (string, error) { // timeout after 10 seconds client := http.Client{Timeout: time.Duration(10 * time.Second)} resp, err := client.Get("http://myexternalip.com/raw") if err != nil { return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { errResp, _ := ioutil.ReadAll(resp.Body) return "", errors.New(string(errResp)) } buf, err := ioutil.ReadAll(io.LimitReader(resp.Body, 64)) if err != nil { return "", err } if len(buf) == 0 { return "", errors.New("myexternalip.com returned a 0 length IP address") } // trim newline return strings.TrimSpace(string(buf)), nil } Sia-1.0.4/modules/host_test.go000066400000000000000000000015011302033070500162310ustar00rootroot00000000000000package modules import ( "testing" ) // TestUnitMaxFileContractSetLenSanity checks that a sensible value for // MaxFileContractSetLen has been chosen. func TestUnitMaxFileContractSetLenSanity(t *testing.T) { t.Parallel() // It does not make sense for the contract set limit to be higher than the // IsStandard limit in the transaction pool. Such a transaction set would // never be accepted by the transaction pool, and therefore is going to // cause a failure later on in the host process. An extra 1kb is left // because the file contract transaction is going to grow as the terms are // negotiated and as signatures are added. if NegotiateMaxFileContractSetLen > TransactionSetSizeLimit-1e3 { t.Fatal("MaxfileContractSetLen does not have a sensible value - should be smaller than the TransactionSetSizeLimit") } } Sia-1.0.4/modules/miner.go000066400000000000000000000051011302033070500153270ustar00rootroot00000000000000package modules import ( "io" "github.com/NebulousLabs/Sia/types" ) const ( // MinerDir is the name of the directory that is used to store the miner's // persistent data. MinerDir = "miner" ) // BlockManager contains functions that can interface with external miners, // providing and receiving blocks that have experienced nonce grinding. type BlockManager interface { // HeaderForWork returns a block header that can be grinded on and // resubmitted to the miner. HeaderForWork() will remember the block that // corresponds to the header for 50 calls. HeaderForWork() (types.BlockHeader, types.Target, error) // SubmitHeader takes a block header that has been worked on and has a // valid target. SubmitHeader(types.BlockHeader) error // BlocksMined returns the number of blocks and stale blocks that have been // mined using this miner. BlocksMined() (goodBlocks, staleBlocks int) } // CPUMiner provides access to a single-threaded cpu miner. type CPUMiner interface { // CPUHashrate returns the hashrate of the cpu miner in hashes per second. CPUHashrate() int // Mining returns true if the cpu miner is enabled, and false otherwise. CPUMining() bool // StartMining turns on the miner, which will endlessly work for new // blocks. StartCPUMining() // StopMining turns off the miner, but keeps the same number of threads. StopCPUMining() } // TestMiner provides direct access to block fetching, solving, and // manipulation. The primary use of this interface is integration testing. type TestMiner interface { // AddBlock is an extension of FindBlock - AddBlock will submit the block // after finding it. AddBlock() (types.Block, error) // BlockForWork returns a block that is ready for nonce grinding. All // blocks returned by BlockForWork have a unique Merkle root, meaning that // each can safely start from nonce 0. BlockForWork() (types.Block, types.Target, error) // Close is necessary for clean shutdown during testing. Close() error // FindBlock will have the miner make 1 attempt to find a solved block that // builds on the current consensus set. It will give up after a few // seconds, returning the block and a bool indicating whether the block is // solved. FindBlock() (types.Block, error) // SolveBlock will have the miner make 1 attempt to solve the input block, // which amounts to trying a few thousand different nonces. SolveBlock is // primarily used for testing. SolveBlock(types.Block, types.Target) (types.Block, bool) } // The Miner interface provides access to mining features. type Miner interface { BlockManager CPUMiner io.Closer } Sia-1.0.4/modules/miner/000077500000000000000000000000001302033070500150035ustar00rootroot00000000000000Sia-1.0.4/modules/miner/blockmanager.go000066400000000000000000000153041302033070500177620ustar00rootroot00000000000000package miner import ( "crypto/rand" "errors" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( errLateHeader = errors.New("header is old, block could not be recovered") ) // blockForWork returns a block that is ready for nonce grinding, including // correct miner payouts and a random transaction to prevent collisions and // overlapping work with other blocks being mined in parallel or for different // forks (during testing). func (m *Miner) blockForWork() types.Block { b := m.persist.UnsolvedBlock // Update the timestmap. if b.Timestamp < types.CurrentTimestamp() { b.Timestamp = types.CurrentTimestamp() } // Update the address + payouts. err := m.checkAddress() if err != nil { m.log.Println(err) } b.MinerPayouts = []types.SiacoinOutput{{Value: b.CalculateSubsidy(m.persist.Height + 1), UnlockHash: m.persist.Address}} // Add an arb-data txn to the block to create a unique merkle root. randBytes, _ := crypto.RandBytes(types.SpecifierLen) randTxn := types.Transaction{ ArbitraryData: [][]byte{append(modules.PrefixNonSia[:], randBytes...)}, } b.Transactions = append([]types.Transaction{randTxn}, b.Transactions...) return b } // newSourceBlock creates a new source block for the block manager so that new // headers will use the updated source block. func (m *Miner) newSourceBlock() { // To guarantee garbage collection of old blocks, delete all header entries // that have not been reached for the current block. for m.memProgress%(HeaderMemory/BlockMemory) != 0 { delete(m.blockMem, m.headerMem[m.memProgress]) delete(m.arbDataMem, m.headerMem[m.memProgress]) m.memProgress++ if m.memProgress == HeaderMemory { m.memProgress = 0 } } // Update the source block. block := m.blockForWork() m.sourceBlock = &block m.sourceBlockTime = time.Now() } // HeaderForWork returns a header that is ready for nonce grinding. The miner // will store the header in memory for a while, depending on the constants // 'HeaderMemory', 'BlockMemory', and 'MaxSourceBlockAge'. On the full network, // it is typically safe to assume that headers will be remembered for // min(10 minutes, 10e3 requests). func (m *Miner) HeaderForWork() (types.BlockHeader, types.Target, error) { if err := m.tg.Add(); err != nil { return types.BlockHeader{}, types.Target{}, err } defer m.tg.Done() m.mu.Lock() defer m.mu.Unlock() // Return a blank header with an error if the wallet is locked. if !m.wallet.Unlocked() { return types.BlockHeader{}, types.Target{}, modules.ErrLockedWallet } // Check that the wallet has been initialized, and that the miner has // successfully fetched an address. err := m.checkAddress() if err != nil { return types.BlockHeader{}, types.Target{}, err } // If too much time has elapsed since the last source block, get a new one. // This typically only happens if the miner has just turned on after being // off for a while. If the current block has been used for too many // requests, fetch a new source block. if time.Since(m.sourceBlockTime) > MaxSourceBlockAge || m.memProgress%(HeaderMemory/BlockMemory) == 0 { m.newSourceBlock() } // Create a header from the source block - this may be a race condition, // but I don't think so (underlying slice may be shared with other blocks // accessible outside the miner). var arbData [crypto.EntropySize]byte _, err = rand.Read(arbData[:]) if err != nil { return types.BlockHeader{}, types.Target{}, err } copy(m.sourceBlock.Transactions[0].ArbitraryData[0], arbData[:]) header := m.sourceBlock.Header() // Save the mapping from the header to its block and from the header to its // arbitrary data, replacing whatever header already exists. delete(m.blockMem, m.headerMem[m.memProgress]) delete(m.arbDataMem, m.headerMem[m.memProgress]) m.blockMem[header] = m.sourceBlock m.arbDataMem[header] = arbData m.headerMem[m.memProgress] = header m.memProgress++ if m.memProgress == HeaderMemory { m.memProgress = 0 } // Return the header and target. return header, m.persist.Target, nil } // managedSubmitBlock takes a solved block and submits it to the blockchain. func (m *Miner) managedSubmitBlock(b types.Block) error { // Give the block to the consensus set. err := m.cs.AcceptBlock(b) // Add the miner to the blocks list if the only problem is that it's stale. if err == modules.ErrNonExtendingBlock { m.mu.Lock() m.persist.BlocksFound = append(m.persist.BlocksFound, b.ID()) m.mu.Unlock() m.log.Println("Mined a stale block - block appears valid but does not extend the blockchain") return err } if err == modules.ErrBlockUnsolved { m.log.Println("Mined an unsolved block - header submission appears to be incorrect") return err } if err != nil { m.tpool.PurgeTransactionPool() m.log.Critical("ERROR: an invalid block was submitted:", err) return err } m.mu.Lock() defer m.mu.Unlock() // Grab a new address for the miner. Call may fail if the wallet is locked // or if the wallet addresses have been exhausted. m.persist.BlocksFound = append(m.persist.BlocksFound, b.ID()) var uc types.UnlockConditions uc, err = m.wallet.NextAddress() if err != nil { return err } m.persist.Address = uc.UnlockHash() return m.saveSync() } // SubmitHeader accepts a block header. func (m *Miner) SubmitHeader(bh types.BlockHeader) error { if err := m.tg.Add(); err != nil { return err } defer m.tg.Done() // Because a call to managedSubmitBlock is required at the end of this // function, the first part needs to be wrapped in an anonymous function // for lock safety. var b types.Block err := func() error { m.mu.Lock() defer m.mu.Unlock() // Lookup the block that corresponds to the provided header. nonce := bh.Nonce bh.Nonce = [8]byte{} bPointer, bExists := m.blockMem[bh] arbData, arbExists := m.arbDataMem[bh] if !bExists || !arbExists { return errLateHeader } // Block is going to be passed to external memory, but the memory pointed // to by the transactions slice is still being modified - needs to be // copied. Same with the memory being pointed to by the arb data slice. b = *bPointer txns := make([]types.Transaction, len(b.Transactions)) copy(txns, b.Transactions) b.Transactions = txns b.Transactions[0].ArbitraryData = [][]byte{arbData[:]} b.Nonce = nonce // Sanity check - block should have same id as header. bh.Nonce = nonce if types.BlockID(crypto.HashObject(bh)) != b.ID() { m.log.Critical("block reconstruction failed") } return nil }() if err != nil { m.log.Println("ERROR during call to SubmitHeader, pre SubmitBlock:", err) return err } err = m.managedSubmitBlock(b) if err != nil { m.log.Println("ERROR returned by managedSubmitBlock:", err) return err } return nil } Sia-1.0.4/modules/miner/blockmanager_test.go000066400000000000000000000131331302033070500210170ustar00rootroot00000000000000package miner import ( "bytes" "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // solveHeader takes a block header as input and returns a solved block header // as output. func solveHeader(header types.BlockHeader, target types.Target) types.BlockHeader { // Solve the header. for { // Increment the nonce first to guarantee that a new header is formed // - this helps check for pointer errors. header.Nonce[0]++ id := crypto.HashObject(header) if bytes.Compare(target[:], id[:]) >= 0 { break } } return header } // TestIntegrationHeaderForWork checks that header requesting, solving, and // submitting naively works. func TestIntegrationHeaderForWork(t *testing.T) { if testing.Short() { t.SkipNow() } mt, err := createMinerTester("TestIntegreationHeaderForWork") if err != nil { t.Fatal(err) } // Get a header and solve it. header, target, err := mt.miner.HeaderForWork() if err != nil { t.Fatal(err) } solvedHeader := solveHeader(header, target) // Sanity check - header and solvedHeader should be different. (within the // testing file, 'header' should always have a nonce of '0' and // solvedHeader should never have a nonce of '0'.) if header.Nonce == solvedHeader.Nonce { t.Fatal("nonce memory is not independent") } // Submit the header. err = mt.miner.SubmitHeader(solvedHeader) if err != nil { t.Fatal(err) } } // TestIntegrationHeaderForWorkUpdates checks that HeaderForWork starts // returning headers on the new block after a block has been submitted to the // consensus set. func TestIntegrationHeaderForWorkUpdates(t *testing.T) { if testing.Short() { t.SkipNow() } mt, err := createMinerTester("TestIntegrationHeaderForWorkUpdates") if err != nil { t.Fatal(err) } // Get a header to advance into the header memory. _, _, err = mt.miner.HeaderForWork() if err != nil { t.Fatal(err) } // Submit a block, which should trigger a header change. _, err = mt.miner.AddBlock() if err != nil { t.Fatal(err) } // Get a header to grind on. header, target, err := mt.miner.HeaderForWork() if err != nil { t.Fatal(err) } solvedHeader := solveHeader(header, target) // Submit the header. err = mt.miner.SubmitHeader(solvedHeader) if err != nil { t.Fatal(err) } if !mt.cs.InCurrentPath(types.BlockID(crypto.HashObject(solvedHeader))) { t.Error("header from solved block is not in the current path") } } // TestIntegrationManyHeaders checks that requesting a full set of headers in a // row results in all unique headers, and that all of them can be reassembled // into valid blocks. func TestIntegrationManyHeaders(t *testing.T) { if testing.Short() { t.SkipNow() } mt, err := createMinerTester("TestIntegrationManyHeaders") if err != nil { t.Fatal(err) } // Create a suite of headers for imaginary parallel mining. solvedHeaders := make([]types.BlockHeader, HeaderMemory/BlockMemory*2) for i := range solvedHeaders { header, target, err := mt.miner.HeaderForWork() if err != nil { t.Fatal(err) } solvedHeaders[i] = solveHeader(header, target) } // Submit the headers randomly and make sure they are all considered valid. selectionOrder, err := crypto.Perm(len(solvedHeaders)) if err != nil { t.Fatal(err) } for _, selection := range selectionOrder { err = mt.miner.SubmitHeader(solvedHeaders[selection]) if err != nil && err != modules.ErrNonExtendingBlock { t.Error(err) } } } // TestIntegrationHeaderBlockOverflow triggers a header overflow by requesting // a block that triggers the overflow. func TestIntegrationHeaderBlockOverflow(t *testing.T) { if testing.Short() { t.SkipNow() } mt, err := createMinerTester("TestIntegrationHeaderBlockOverflow") if err != nil { t.Fatal(err) } // Grab a header that will be overwritten. header, target, err := mt.miner.HeaderForWork() if err != nil { t.Fatal(err) } header = solveHeader(header, target) // Mine blocks to wrap the memProgress around and wipe the old header. for i := 0; i < BlockMemory; i++ { _, err = mt.miner.AddBlock() if err != nil { t.Fatal(err) } // Grab a header to advance the mempool progress. _, _, err = mt.miner.HeaderForWork() if err != nil { t.Fatal(err) } } // Previous header should no longer be in memory. err = mt.miner.SubmitHeader(header) if err != errLateHeader { t.Error(err) } } // TestIntegrationHeaderRequestOverflow triggers a header overflow by // requesting a header that triggers overflow. func TestIntegrationHeaderRequestOverflow(t *testing.T) { if testing.Short() { t.SkipNow() } mt, err := createMinerTester("TestIntegrationHeaderRequestOverflow") if err != nil { t.Fatal(err) } // Grab a header that will be overwritten. header, target, err := mt.miner.HeaderForWork() if err != nil { t.Fatal(err) } header = solveHeader(header, target) // Mine blocks to bring memProgress up to the edge. The number is chosen // specifically so that the overflow happens during the requesting of 200 // headers. for i := 0; i < BlockMemory-1; i++ { _, err = mt.miner.AddBlock() if err != nil { t.Fatal(err) } // Grab a header to advance the mempool progress. _, _, err = mt.miner.HeaderForWork() if err != nil { t.Fatal(err) } } // Header should still be in memory. err = mt.miner.SubmitHeader(header) if err != modules.ErrNonExtendingBlock { t.Error(err) } // Request headers until the overflow is achieved. for i := 0; i < HeaderMemory/BlockMemory; i++ { _, _, err = mt.miner.HeaderForWork() if err != nil { t.Fatal(err) } } err = mt.miner.SubmitHeader(header) if err != errLateHeader { t.Error(err) } } Sia-1.0.4/modules/miner/cpuminer.go000066400000000000000000000053601302033070500171600ustar00rootroot00000000000000package miner import ( "time" "github.com/NebulousLabs/Sia/build" ) // threadedMine starts a gothread that does CPU mining. threadedMine is the // only function that should be setting the mining flag to true. func (m *Miner) threadedMine() { if err := m.tg.Add(); err != nil { return } defer m.tg.Done() // There should not be another thread mining, and mining should be enabled. m.mu.Lock() if m.mining || !m.miningOn { m.mu.Unlock() return } m.mining = true m.mu.Unlock() // Solve blocks repeatedly, keeping track of how fast hashing is // occurring. cycleStart := time.Now() for { m.mu.Lock() // Kill the thread if 'Stop' has been called. select { case <-m.tg.StopChan(): m.miningOn = false m.mining = false m.mu.Unlock() return default: } // Kill the thread if mining has been turned off. if !m.miningOn { m.mining = false m.mu.Unlock() return } // Prepare the work and release the miner lock. bfw := m.blockForWork() target := m.persist.Target m.mu.Unlock() // Solve the block. b, solved := solveBlock(bfw, target) if solved { err := m.managedSubmitBlock(b) if err != nil { m.log.Println("ERROR: An error occurred while cpu mining:", err) } } // Update the hashrate. If the block was solved, the full set of // iterations was not completed, so the hashrate should not be updated. m.mu.Lock() if !solved { nanosecondsElapsed := 1 + time.Since(cycleStart).Nanoseconds() // Add 1 to prevent divide by zero errors. cycleStart = time.Now() // Reset the cycle counter as soon as the previous value is measured. m.hashRate = 1e9 * solveAttempts / nanosecondsElapsed } m.mu.Unlock() } } // CPUHashrate returns an estimated cpu hashrate. func (m *Miner) CPUHashrate() int { if err := m.tg.Add(); err != nil { build.Critical(err) } defer m.tg.Done() m.mu.Lock() defer m.mu.Unlock() return int(m.hashRate) } // CPUMining indicates whether the cpu miner is running. func (m *Miner) CPUMining() bool { if err := m.tg.Add(); err != nil { build.Critical(err) } defer m.tg.Done() m.mu.Lock() defer m.mu.Unlock() return m.miningOn } // StartCPUMining will start a single threaded cpu miner. If the miner is // already running, nothing will happen. func (m *Miner) StartCPUMining() { if err := m.tg.Add(); err != nil { build.Critical(err) } defer m.tg.Done() m.mu.Lock() defer m.mu.Unlock() m.miningOn = true go m.threadedMine() } // StopCPUMining will stop the cpu miner. If the cpu miner is already stopped, // nothing will happen. func (m *Miner) StopCPUMining() { if err := m.tg.Add(); err != nil { build.Critical(err) } defer m.tg.Done() m.mu.Lock() defer m.mu.Unlock() m.hashRate = 0 m.miningOn = false } Sia-1.0.4/modules/miner/miner.go000066400000000000000000000177731302033070500164630ustar00rootroot00000000000000package miner import ( "errors" "fmt" "sync" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" siasync "github.com/NebulousLabs/Sia/sync" "github.com/NebulousLabs/Sia/types" ) var ( errNilCS = errors.New("miner cannot use a nil consensus set") errNilTpool = errors.New("miner cannot use a nil transaction pool") errNilWallet = errors.New("miner cannot use a nil wallet") // HeaderMemory is the number of previous calls to 'header' // that are remembered. Additionally, 'header' will only poll for a // new block every 'headerMemory / blockMemory' times it is // called. This reduces the amount of memory used, but comes at the cost of // not always having the most recent transactions. HeaderMemory = func() int { if build.Release == "dev" { return 500 } if build.Release == "standard" { return 10000 } if build.Release == "testing" { return 50 } panic("unrecognized build.Release") }() // BlockMemory is the maximum number of blocks the miner will store // Blocks take up to 2 megabytes of memory, which is why this number is // limited. BlockMemory = func() int { if build.Release == "dev" { return 10 } if build.Release == "standard" { return 50 } if build.Release == "testing" { return 5 } panic("unrecognized build.Release") }() // MaxSourceBlockAge is the maximum amount of time that is allowed to // elapse between generating source blocks. MaxSourceBlockAge = func() time.Duration { if build.Release == "dev" { return 5 * time.Second } if build.Release == "standard" { return 30 * time.Second } if build.Release == "testing" { return 1 * time.Second } panic("unrecognized build.Release") }() ) // Miner struct contains all variables the miner needs // in order to create and submit blocks. type Miner struct { // Module dependencies. cs modules.ConsensusSet tpool modules.TransactionPool wallet modules.Wallet // BlockManager variables. Becaues blocks are large, one block is used to // make many headers which can be used by miners. Headers include an // arbitrary data transaction (appended to the block) to make the merkle // roots unique (preventing miners from doing redundant work). Every N // requests or M seconds, a new block is used to create headers. // // Only 'blocksMemory' blocks are kept in memory at a time, which // keeps ram usage reasonable. Miners may request many headers in parallel, // and thus may be working on different blocks. When they submit the solved // header to the block manager, the rest of the block needs to be found in // a lookup. blockMem map[types.BlockHeader]*types.Block // Mappings from headers to the blocks they are derived from. arbDataMem map[types.BlockHeader][crypto.EntropySize]byte // Mappings from the headers to their unique arb data. headerMem []types.BlockHeader // A circular list of headers that have been given out from the api recently. sourceBlock *types.Block // The block from which new headers for mining are created. sourceBlockTime time.Time // How long headers have been using the same block (different from 'recent block'). memProgress int // The index of the most recent header used in headerMem. // CPUMiner variables. miningOn bool // indicates if the miner is supposed to be running mining bool // indicates if the miner is actually running hashRate int64 // indicates hashes per second // Utils log *persist.Logger mu sync.RWMutex persist persistence persistDir string // tg signals the Miner's goroutines to shut down and blocks until all // goroutines have exited before returning from Close(). tg siasync.ThreadGroup } // startupRescan will rescan the blockchain in the event that the miner // persistance layer has become desynchronized from the consensus persistance // layer. This might happen if a user replaces any of the folders with backups // or deletes any of the folders. func (m *Miner) startupRescan() error { // Reset all of the variables that have relevance to the consensus set. The // operations are wrapped by an anonymous function so that the locking can // be handled using a defer statement. err := func() error { m.mu.Lock() defer m.mu.Unlock() m.log.Println("Performing a miner rescan.") m.persist.RecentChange = modules.ConsensusChangeBeginning m.persist.Height = 0 m.persist.Target = types.Target{} return m.save() }() if err != nil { return err } // Subscribe to the consensus set. This is a blocking call that will not // return until the miner has fully caught up to the current block. err = m.cs.ConsensusSetSubscribe(m, modules.ConsensusChangeBeginning) if err != nil { return err } m.tg.OnStop(func() { m.cs.Unsubscribe(m) }) return nil } // New returns a ready-to-go miner that is not mining. func New(cs modules.ConsensusSet, tpool modules.TransactionPool, w modules.Wallet, persistDir string) (*Miner, error) { // Create the miner and its dependencies. if cs == nil { return nil, errNilCS } if tpool == nil { return nil, errNilTpool } if w == nil { return nil, errNilWallet } // Assemble the miner. The miner is assembled without an address because // the wallet is likely not unlocked yet. The miner will grab an address // after the miner is unlocked (this must be coded manually for each // function that potentially requires the miner to have an address. m := &Miner{ cs: cs, tpool: tpool, wallet: w, blockMem: make(map[types.BlockHeader]*types.Block), arbDataMem: make(map[types.BlockHeader][crypto.EntropySize]byte), headerMem: make([]types.BlockHeader, HeaderMemory), persistDir: persistDir, } err := m.initPersist() if err != nil { return nil, errors.New("miner persistence startup failed: " + err.Error()) } err = m.cs.ConsensusSetSubscribe(m, m.persist.RecentChange) if err == modules.ErrInvalidConsensusChangeID { // Perform a rescan of the consensus set if the change id is not found. // The id will only be not found if there has been desynchronization // between the miner and the consensus package. err = m.startupRescan() if err != nil { return nil, errors.New("miner startup failed - rescanning failed: " + err.Error()) } } else if err != nil { return nil, errors.New("miner subscription failed: " + err.Error()) } m.tg.OnStop(func() { m.cs.Unsubscribe(m) }) m.tpool.TransactionPoolSubscribe(m) m.tg.OnStop(func() { m.tpool.Unsubscribe(m) }) // Save after synchronizing with consensus err = m.save() if err != nil { return nil, errors.New("miner could not save during startup: " + err.Error()) } return m, nil } // Close terminates all ongoing processes involving the miner, enabling garbage // collection. func (m *Miner) Close() error { if err := m.tg.Stop(); err != nil { return err } m.mu.Lock() defer m.mu.Unlock() m.cs.Unsubscribe(m) var errs []error if err := m.saveSync(); err != nil { errs = append(errs, fmt.Errorf("save failed: %v", err)) } if err := m.log.Close(); err != nil { errs = append(errs, fmt.Errorf("log.Close failed: %v", err)) } return build.JoinErrors(errs, "; ") } // checkAddress checks that the miner has an address, fetching an address from // the wallet if not. func (m *Miner) checkAddress() error { if m.persist.Address != (types.UnlockHash{}) { return nil } uc, err := m.wallet.NextAddress() if err != nil { return err } m.persist.Address = uc.UnlockHash() return nil } // BlocksMined returns the number of good blocks and stale blocks that have // been mined by the miner. func (m *Miner) BlocksMined() (goodBlocks, staleBlocks int) { if err := m.tg.Add(); err != nil { build.Critical(err) } defer m.tg.Done() m.mu.Lock() defer m.mu.Unlock() for _, blockID := range m.persist.BlocksFound { if m.cs.InCurrentPath(blockID) { goodBlocks++ } else { staleBlocks++ } } return } Sia-1.0.4/modules/miner/miner_test.go000066400000000000000000000220611302033070500175040ustar00rootroot00000000000000package miner import ( "bytes" "crypto/rand" "path/filepath" "testing" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/consensus" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/modules/transactionpool" "github.com/NebulousLabs/Sia/modules/wallet" "github.com/NebulousLabs/Sia/types" ) // A minerTester is the helper object for miner testing. type minerTester struct { gateway modules.Gateway cs modules.ConsensusSet tpool modules.TransactionPool wallet modules.Wallet walletKey crypto.TwofishKey miner *Miner minedBlocks []types.Block persistDir string } // createMinerTester creates a minerTester that's ready for use. func createMinerTester(name string) (*minerTester, error) { testdir := build.TempDir(modules.MinerDir, name) // Create the modules. g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, err } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { return nil, err } w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } var key crypto.TwofishKey _, err = rand.Read(key[:]) if err != nil { return nil, err } _, err = w.Encrypt(key) if err != nil { return nil, err } err = w.Unlock(key) if err != nil { return nil, err } m, err := New(cs, tp, w, filepath.Join(testdir, modules.MinerDir)) if err != nil { return nil, err } // Assemble the minerTester. mt := &minerTester{ gateway: g, cs: cs, tpool: tp, wallet: w, walletKey: key, miner: m, persistDir: testdir, } // Mine until the wallet has money. for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ { b, err := m.AddBlock() if err != nil { return nil, err } mt.minedBlocks = append(mt.minedBlocks, b) } return mt, nil } // TestIntegrationMiner creates a miner, mines a few blocks, and checks that // the wallet balance is updating as the blocks get mined. func TestIntegrationMiner(t *testing.T) { if testing.Short() { t.SkipNow() } mt, err := createMinerTester("TestMiner") if err != nil { t.Fatal(err) } // Check that the wallet has money. siacoins, _, _ := mt.wallet.ConfirmedBalance() if siacoins.IsZero() { t.Error("expecting mining full balance to not be zero") } // Mine a bunch of blocks. for i := 0; i < 50; i++ { b, _ := mt.miner.FindBlock() err = mt.cs.AcceptBlock(b) if err != nil { t.Fatal(err) } } morecoins, _, _ := mt.wallet.ConfirmedBalance() if siacoins.Cmp(morecoins) >= 0 { t.Error("wallet is not gaining balance while mining") } } // TestIntegrationNilMinerDependencies tests that the miner properly handles // nil inputs for its dependencies. func TestIntegrationNilMinerDependencies(t *testing.T) { if testing.Short() { t.SkipNow() } mt, err := createMinerTester("TestIntegrationNilMinerDependencies") if err != nil { t.Fatal(err) } _, err = New(mt.cs, mt.tpool, nil, "") if err != errNilWallet { t.Fatal(err) } _, err = New(mt.cs, nil, mt.wallet, "") if err != errNilTpool { t.Fatal(err) } _, err = New(nil, mt.tpool, mt.wallet, "") if err != errNilCS { t.Fatal(err) } _, err = New(nil, nil, nil, "") if err == nil { t.Fatal(err) } } // TestIntegrationBlocksMined checks that the BlocksMined function correctly // indicates the number of real blocks and stale blocks that have been mined. func TestIntegrationBlocksMined(t *testing.T) { if testing.Short() { t.SkipNow() } mt, err := createMinerTester("TestIntegrationBlocksMined") if err != nil { t.Fatal(err) } // Get an unsolved header. unsolvedHeader, target, err := mt.miner.HeaderForWork() if err != nil { t.Fatal(err) } // Unsolve the header - necessary because the target is very low when // mining. for { unsolvedHeader.Nonce[0]++ id := crypto.HashObject(unsolvedHeader) if bytes.Compare(target[:], id[:]) < 0 { break } } // Get two solved headers. header1, target, err := mt.miner.HeaderForWork() if err != nil { t.Fatal(err) } header1 = solveHeader(header1, target) header2, target, err := mt.miner.HeaderForWork() if err != nil { t.Fatal(err) } header2 = solveHeader(header2, target) // Submit the unsolved header followed by the two solved headers, this // should result in 1 real block mined and 1 stale block mined. err = mt.miner.SubmitHeader(unsolvedHeader) if err != modules.ErrBlockUnsolved { t.Fatal(err) } err = mt.miner.SubmitHeader(header1) if err != nil { t.Fatal(err) } err = mt.miner.SubmitHeader(header2) if err != modules.ErrNonExtendingBlock { t.Fatal(err) } goodBlocks, staleBlocks := mt.miner.BlocksMined() if goodBlocks != 1 { t.Error("expecting 1 good block") } if staleBlocks != 1 { t.Error("expecting 1 stale block, got", staleBlocks) } // Reboot the miner and verify that the block record has persisted. err = mt.miner.Close() if err != nil { t.Fatal(err) } rebootMiner, err := New(mt.cs, mt.tpool, mt.wallet, filepath.Join(mt.persistDir, modules.MinerDir)) if err != nil { t.Fatal(err) } goodBlocks, staleBlocks = rebootMiner.BlocksMined() if goodBlocks != 1 { t.Error("expecting 1 good block") } if staleBlocks != 1 { t.Error("expecting 1 stale block, got", staleBlocks) } } // TestIntegrationAutoRescan triggers a rescan during a call to New and // verifies that the rescanning happens correctly. The rescan is triggered by // a call to New, instead of getting called directly. func TestIntegrationAutoRescan(t *testing.T) { if testing.Short() { t.SkipNow() } mt, err := createMinerTester("TestIntegrationAutoRescan") if err != nil { t.Fatal(err) } _, err = mt.miner.AddBlock() if err != nil { t.Fatal(err) } // Get the persist data of the current miner. oldChange := mt.miner.persist.RecentChange oldHeight := mt.miner.persist.Height oldTarget := mt.miner.persist.Target // Corrupt the miner, close the miner, and make a new one from the same // directory. mt.miner.persist.RecentChange[0]++ mt.miner.persist.Height += 1e5 mt.miner.persist.Target[0]++ err = mt.miner.Close() // miner saves when it closes. if err != nil { t.Fatal(err) } // Verify that rescanning resolved the corruption in the miner. m, err := New(mt.cs, mt.tpool, mt.wallet, filepath.Join(mt.persistDir, modules.MinerDir)) if err != nil { t.Fatal(err) } // Check that after rescanning, the values have returned to the usual values. if m.persist.RecentChange != oldChange { t.Error("rescan failed, ended up on the wrong change") } if m.persist.Height != oldHeight { t.Error("rescan failed, ended up at the wrong height") } if m.persist.Target != oldTarget { t.Error("rescan failed, ended up at the wrong target") } } // TestIntegrationStartupRescan probes the startupRescan function, checking // that it works in the naive case. Rescan is called directly. func TestIntegrationStartupRescan(t *testing.T) { if testing.Short() { t.SkipNow() } mt, err := createMinerTester("TestIntegrationStartupRescan") if err != nil { t.Fatal(err) } // Check that the miner's persist variables have been initialized to the // first few blocks. if mt.miner.persist.RecentChange == (modules.ConsensusChangeID{}) || mt.miner.persist.Height == 0 || mt.miner.persist.Target == (types.Target{}) { t.Fatal("miner persist variables not initialized") } oldChange := mt.miner.persist.RecentChange oldHeight := mt.miner.persist.Height oldTarget := mt.miner.persist.Target // Corrupt the miner and verify that a rescan repairs the corruption. mt.miner.persist.RecentChange[0]++ mt.miner.persist.Height += 500 mt.miner.persist.Target[0]++ mt.cs.Unsubscribe(mt.miner) err = mt.miner.startupRescan() if err != nil { t.Fatal(err) } if mt.miner.persist.RecentChange != oldChange { t.Error("rescan failed, ended up on the wrong change") } if mt.miner.persist.Height != oldHeight { t.Error("rescan failed, ended up at the wrong height") } if mt.miner.persist.Target != oldTarget { t.Error("rescan failed, ended up at the wrong target") } } // TestMinerCloseDeadlock checks that the miner can cleanly close even if the // CPU miner is running. func TestMinerCloseDeadlock(t *testing.T) { mt, err := createMinerTester("TestMinerCloseDeadlock") if err != nil { t.Fatal(err) } // StartCPUMining calls `go threadedMine()`, which needs to access the miner // before Close() does in the next goroutine, otherwise m.tg.Add() fails // at the top of threadedMine() and threadedMine() exits (silently!). // I haven't seen this behavior since sticking Close() inside a goroutine, // but I'm not sure that's comfort enough. mt.miner.StartCPUMining() time.Sleep(time.Millisecond * 250) closed := make(chan struct{}) go func() { if err := mt.miner.Close(); err != nil { t.Fatal(err) } closed <- struct{}{} }() select { case <-closed: case <-time.After(5 * time.Second): t.Fatal("mt.miner.Close never completed") } } Sia-1.0.4/modules/miner/persist.go000066400000000000000000000035321302033070500170260ustar00rootroot00000000000000package miner import ( "os" "path/filepath" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/types" ) const ( logFile = modules.MinerDir + ".log" settingsFile = modules.MinerDir + ".json" ) var ( settingsMetadata = persist.Metadata{ Header: "Miner Settings", Version: "0.5.0", } ) type ( // persist contains all of the persistent miner data. persistence struct { RecentChange modules.ConsensusChangeID Height types.BlockHeight Target types.Target Address types.UnlockHash BlocksFound []types.BlockID UnsolvedBlock types.Block } ) // initSettings loads the settings file if it exists and creates it if it // doesn't. func (m *Miner) initSettings() error { filename := filepath.Join(m.persistDir, settingsFile) _, err := os.Stat(filename) if os.IsNotExist(err) { return m.save() } else if err != nil { return err } return m.load() } // initPersist initializes the persistence of the miner. func (m *Miner) initPersist() error { // Create the miner directory. err := os.MkdirAll(m.persistDir, 0700) if err != nil { return err } // Add a logger. m.log, err = persist.NewFileLogger(filepath.Join(m.persistDir, logFile)) if err != nil { return err } return m.initSettings() } // load loads the miner persistence from disk. func (m *Miner) load() error { return persist.LoadFile(settingsMetadata, &m.persist, filepath.Join(m.persistDir, settingsFile)) } // save saves the miner persistence to disk. func (m *Miner) save() error { return persist.SaveFile(settingsMetadata, m.persist, filepath.Join(m.persistDir, settingsFile)) } // saveSync saves the miner persistence to disk, and then syncs to disk. func (m *Miner) saveSync() error { return persist.SaveFileSync(settingsMetadata, m.persist, filepath.Join(m.persistDir, settingsFile)) } Sia-1.0.4/modules/miner/testminer.go000066400000000000000000000055231302033070500173510ustar00rootroot00000000000000package miner // testminer.go implements the TestMiner interface, whose primary purpose is // integration testing. import ( "bytes" "encoding/binary" "errors" "unsafe" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) const ( // solveAttempts is the number of times that SolveBlock will try to solve a // block before giving up. solveAttempts = 16e3 ) // solveBlock takes a block and a target and tries to solve the block for the // target. A bool is returned indicating whether the block was successfully // solved. func solveBlock(b types.Block, target types.Target) (types.Block, bool) { // Assemble the header. merkleRoot := b.MerkleRoot() header := make([]byte, 80) copy(header, b.ParentID[:]) binary.LittleEndian.PutUint64(header[40:48], uint64(b.Timestamp)) copy(header[48:], merkleRoot[:]) var nonce uint64 for i := 0; i < solveAttempts; i++ { id := crypto.HashBytes(header) if bytes.Compare(target[:], id[:]) >= 0 { copy(b.Nonce[:], header[32:40]) return b, true } *(*uint64)(unsafe.Pointer(&header[32])) = nonce nonce++ } return b, false } // BlockForWork returns a block that is ready for nonce grinding, along with // the root hash of the block. func (m *Miner) BlockForWork() (b types.Block, t types.Target, err error) { // Check if the wallet is unlocked. If the wallet is unlocked, make sure // that the miner has a recent address. if !m.wallet.Unlocked() { err = modules.ErrLockedWallet return } m.mu.Lock() defer m.mu.Unlock() err = m.checkAddress() if err != nil { return } b = m.blockForWork() return b, m.persist.Target, nil } // AddBlock adds a block to the consensus set. func (m *Miner) AddBlock() (types.Block, error) { block, err := m.FindBlock() if err != nil { return types.Block{}, err } err = m.cs.AcceptBlock(block) if err != nil { return types.Block{}, err } return block, nil } // FindBlock finds at most one block that extends the current blockchain. func (m *Miner) FindBlock() (types.Block, error) { var bfw types.Block var target types.Target err := func() error { m.mu.Lock() defer m.mu.Unlock() if !m.wallet.Unlocked() { return modules.ErrLockedWallet } err := m.checkAddress() if err != nil { return err } // Get a block for work. bfw = m.blockForWork() target = m.persist.Target return nil }() if err != nil { return types.Block{}, err } block, ok := m.SolveBlock(bfw, target) if !ok { return types.Block{}, errors.New("could not solve block using limited hashing power") } return block, nil } // SolveBlock takes a block and a target and tries to solve the block for the // target. A bool is returned indicating whether the block was successfully // solved. func (m *Miner) SolveBlock(b types.Block, target types.Target) (types.Block, bool) { return solveBlock(b, target) } Sia-1.0.4/modules/miner/update.go000066400000000000000000000053751302033070500166260ustar00rootroot00000000000000package miner import ( "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // ProcessConsensusDigest will update the miner's most recent block. func (m *Miner) ProcessConsensusChange(cc modules.ConsensusChange) { m.mu.Lock() defer m.mu.Unlock() // Update the miner's understanding of the block height. for _, block := range cc.RevertedBlocks { // Only doing the block check if the height is above zero saves hashing // and saves a nontrivial amount of time during IBD. if m.persist.Height > 0 || block.ID() != types.GenesisID { m.persist.Height-- } else if m.persist.Height != 0 { // Sanity check - if the current block is the genesis block, the // miner height should be set to zero. m.log.Critical("Miner has detected a genesis block, but the height of the miner is set to ", m.persist.Height) m.persist.Height = 0 } } for _, block := range cc.AppliedBlocks { // Only doing the block check if the height is above zero saves hashing // and saves a nontrivial amount of time during IBD. if m.persist.Height > 0 || block.ID() != types.GenesisID { m.persist.Height++ } else if m.persist.Height != 0 { // Sanity check - if the current block is the genesis block, the // miner height should be set to zero. m.log.Critical("Miner has detected a genesis block, but the height of the miner is set to ", m.persist.Height) m.persist.Height = 0 } } // Update the unsolved block. m.persist.UnsolvedBlock.ParentID = cc.AppliedBlocks[len(cc.AppliedBlocks)-1].ID() m.persist.Target = cc.ChildTarget m.persist.UnsolvedBlock.Timestamp = cc.MinimumValidChildTimestamp // There is a new parent block, the source block should be updated to keep // the stale rate as low as possible. m.newSourceBlock() m.persist.RecentChange = cc.ID err := m.save() if err != nil { m.log.Println(err) } } // ReceiveUpdatedUnconfirmedTransactions will replace the current unconfirmed // set of transactions with the input transactions. func (m *Miner) ReceiveUpdatedUnconfirmedTransactions(unconfirmedTransactions []types.Transaction, _ modules.ConsensusChange) { m.mu.Lock() defer m.mu.Unlock() // Edge case - if there are no transactions, set the block's transactions // to nil and return. if len(unconfirmedTransactions) == 0 { m.persist.UnsolvedBlock.Transactions = nil return } // Add transactions to the block until the block size limit is reached. // Transactions are assumed to be in a sensible order. var i int remainingSize := int(types.BlockSizeLimit - 5e3) for i = range unconfirmedTransactions { remainingSize -= len(encoding.Marshal(unconfirmedTransactions[i])) if remainingSize < 0 { break } } m.persist.UnsolvedBlock.Transactions = unconfirmedTransactions[:i+1] } Sia-1.0.4/modules/miner/update_test.go000066400000000000000000000051031302033070500176520ustar00rootroot00000000000000package miner import ( "testing" "github.com/NebulousLabs/Sia/modules" ) // TestIntegrationBlockHeightReorg checks that the miner has the correct block // height after a series of reorgs that go as far as the genesis block. func TestIntegrationBlockHeightReorg(t *testing.T) { if testing.Short() { t.SkipNow() } // Create 3 miner testers that will be used to cause each other to reorg. mt1, err := createMinerTester("TestIntegrationBlockHeightReorg - 1") if err != nil { t.Fatal(err) } mt2, err := createMinerTester("TestIntegrationBlockHeightReorg - 2") if err != nil { t.Fatal(err) } mt3, err := createMinerTester("TestIntegrationBlockHeightReorg - 3") if err != nil { t.Fatal(err) } // Put one ahead of the other multiple times, which should thrash around // the height calculation and cause problems by dipping down to the genesis // block repeatedly. for i := 0; i < 2; i++ { b, err := mt1.miner.AddBlock() if err != nil { t.Fatal(err) } mt1.minedBlocks = append(mt1.minedBlocks, b) } for i := 0; i < 3; i++ { b, err := mt2.miner.AddBlock() if err != nil { t.Fatal(err) } mt2.minedBlocks = append(mt2.minedBlocks, b) } for _, block := range mt2.minedBlocks { err = mt1.cs.AcceptBlock(block) if err != nil && err != modules.ErrNonExtendingBlock { t.Fatal(err) } } if mt1.cs.CurrentBlock().ID() != mt2.cs.CurrentBlock().ID() { t.Fatal("mt1 and mt2 should have the same current block") } for i := 0; i < 2; i++ { b, err := mt1.miner.AddBlock() if err != nil { t.Fatal(err) } mt1.minedBlocks = append(mt1.minedBlocks, b) } for i := 0; i < 3; i++ { b, err := mt2.miner.AddBlock() if err != nil { t.Fatal(err) } mt2.minedBlocks = append(mt2.minedBlocks, b) } for _, block := range mt2.minedBlocks { err = mt1.cs.AcceptBlock(block) if err != nil && err != modules.ErrNonExtendingBlock && err != modules.ErrBlockKnown { t.Fatal(err) } } if mt1.cs.CurrentBlock().ID() != mt2.cs.CurrentBlock().ID() { t.Fatal("mt1 and mt2 should have the same current block") } for i := 0; i < 7; i++ { b, err := mt3.miner.AddBlock() if err != nil { t.Fatal(err) } mt3.minedBlocks = append(mt3.minedBlocks, b) } for _, block := range mt3.minedBlocks { err = mt1.cs.AcceptBlock(block) if err != nil && err != modules.ErrNonExtendingBlock { t.Fatal(err) } } if mt1.cs.CurrentBlock().ID() == mt2.cs.CurrentBlock().ID() { t.Fatal("mt1 and mt2 should not have the same block height") } if mt1.cs.CurrentBlock().ID() != mt3.cs.CurrentBlock().ID() { t.Fatal("mt1 and mt3 should have the same current block") } } Sia-1.0.4/modules/modules.go000066400000000000000000000014361302033070500156740ustar00rootroot00000000000000// Package modules contains definitions for all of the major modules of Sia, as // well as some helper functions for performing actions that are common to // multiple modules. package modules import ( "time" "github.com/NebulousLabs/Sia/build" ) var ( // SafeMutexDelay is the recommended timeout for the deadlock detecting // mutex. This value is DEPRECATED, as safe mutexes are no longer // recommended. Instead, the locking conventions should be followed and a // traditional mutex or a demote mutex should be used. SafeMutexDelay time.Duration ) func init() { if build.Release == "dev" { SafeMutexDelay = 60 * time.Second } else if build.Release == "standard" { SafeMutexDelay = 90 * time.Second } else if build.Release == "testing" { SafeMutexDelay = 30 * time.Second } } Sia-1.0.4/modules/negotiate.go000066400000000000000000000424251302033070500162060ustar00rootroot00000000000000package modules import ( "bytes" "errors" "io" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/types" ) const ( // AcceptResponse is the response given to an RPC call to indicate // acceptance, i.e. that the sender wishes to continue communication. AcceptResponse = "accept" // StopResponse is the response given to an RPC call to indicate graceful // termination, i.e. that the sender wishes to cease communication, but // not due to an error. StopResponse = "stop" // NegotiateDownloadTime defines the amount of time that the renter and // host have to negotiate a download request batch. The time is set high // enough that two nodes behind Tor have a reasonable chance of completing // the negotiation. NegotiateDownloadTime = 600 * time.Second // NegotiateFileContractTime defines the amount of time that the renter and // host have to negotiate a file contract. The time is set high enough that // a node behind Tor has a reasonable chance at making the multiple // required round trips to complete the negotiation. NegotiateFileContractTime = 360 * time.Second // NegotiateFileContractRevisionTime defines the minimum amount of time // that the renter and host have to negotiate a file contract revision. The // time is set high enough that a full 4MB can be piped through a // connection that is running over Tor. NegotiateFileContractRevisionTime = 600 * time.Second // NegotiateRecentRevisionTime establishes the minimum amount of time that // the connection deadline is expected to be set to when a recent file // contract revision is being requested from the host. The deadline is long // enough that the connection should be successful even if both parties are // running Tor. NegotiateRecentRevisionTime = 120 * time.Second // NegotiateRenewContractTime defines the minimum amount of time that the // renter and host have to negotiate a final contract renewal. The time is // high enough that the negotiation can occur over a Tor connection, and // that both the host and the renter can have time to process large Merkle // tree calculations that may be involved with renewing a file contract. NegotiateRenewContractTime = 600 * time.Second // NegotiateSettingsTime establishes the minimum amount of time that the // connection deadline is expected to be set to when settings are being // requested from the host. The deadline is long enough that the connection // should be successful even if both parties are on Tor. NegotiateSettingsTime = 120 * time.Second // NegotiateMaxDownloadActionRequestSize defines the maximum size that a // download request can be. Note, this is not a max size for the data that // can be requested, but instead is a max size for the definition of the // data being requested. NegotiateMaxDownloadActionRequestSize = 50e3 // NegotiateMaxErrorSize indicates the maximum number of bytes that can be // used to encode an error being sent during negotiation. NegotiateMaxErrorSize = 256 // NegotiateMaxFileContractRevisionSize specifies the maximum size that a // file contract revision is allowed to have when being sent over the wire // during negotiation. NegotiateMaxFileContractRevisionSize = 3e3 // NegotiateMaxFileContractSetLen determines the maximum allowed size of a // transaction set that can be sent when trying to negotiate a file // contract. The transaction set will contain all of the unconfirmed // dependencies of the file contract, meaning that it can be quite large. // The transaction pool's size limit for transaction sets has been chosen // as a reasonable guideline for determining what is too large. NegotiateMaxFileContractSetLen = TransactionSetSizeLimit - 1e3 // NegotiateMaxHostExternalSettingsLen is the maximum allowed size of an // encoded HostExternalSettings. NegotiateMaxHostExternalSettingsLen = 16000 // NegotiateMaxSiaPubkeySize defines the maximum size that a SiaPubkey is // allowed to be when being sent over the wire during negotiation. NegotiateMaxSiaPubkeySize = 1e3 // NegotiateMaxTransactionSignatureSize defines the maximum size that a // transaction signature is allowed to be when being sent over the wire // during negoitation. NegotiateMaxTransactionSignatureSize = 2e3 // NegotiateMaxTransactionSignaturesSize defines the maximum size that a // transaction signature slice is allowed to be when being sent over the // wire during negoitation. NegotiateMaxTransactionSignaturesSize = 5e3 ) var ( // ActionDelete is the specifier for a RevisionAction that deletes a // sector. ActionDelete = types.Specifier{'D', 'e', 'l', 'e', 't', 'e'} // ActionInsert is the specifier for a RevisionAction that inserts a // sector. ActionInsert = types.Specifier{'I', 'n', 's', 'e', 'r', 't'} // ActionModify is the specifier for a RevisionAction that modifies sector // data. ActionModify = types.Specifier{'M', 'o', 'd', 'i', 'f', 'y'} // ErrAnnNotAnnouncement indicates that the provided host announcement does // not use a recognized specifier, indicating that it's either not a host // announcement or it's not a recognized version of a host announcement. ErrAnnNotAnnouncement = errors.New("provided data does not form a recognized host announcement") // ErrAnnUnrecognizedSignature is returned when the signature in a host // announcement is not a type of signature that is recognized. ErrAnnUnrecognizedSignature = errors.New("the signature provided in the host announcement is not recognized") // ErrRevisionCoveredFields is returned if there is a covered fields object // in a transaction signature which has the 'WholeTransaction' field set to // true, meaning that miner fees cannot be added to the transaction without // invalidating the signature. ErrRevisionCoveredFields = errors.New("file contract revision transaction signature does not allow miner fees to be added") // ErrRevisionSigCount is returned when a file contract revision has the // wrong number of transaction signatures. ErrRevisionSigCount = errors.New("file contract revision has the wrong number of transaction signatures") // ErrStopResponse is the error returned by ReadNegotiationAcceptance when // it reads the StopResponse string. ErrStopResponse = errors.New("sender wishes to stop communicating") // PrefixHostAnnouncement is used to indicate that a transaction's // Arbitrary Data field contains a host announcement. The encoded // announcement will follow this prefix. PrefixHostAnnouncement = types.Specifier{'H', 'o', 's', 't', 'A', 'n', 'n', 'o', 'u', 'n', 'c', 'e', 'm', 'e', 'n', 't'} // RPCDownload is the specifier for downloading a file from a host. RPCDownload = types.Specifier{'D', 'o', 'w', 'n', 'l', 'o', 'a', 'd', 2} // RPCFormContract is the specifier for forming a contract with a host. RPCFormContract = types.Specifier{'F', 'o', 'r', 'm', 'C', 'o', 'n', 't', 'r', 'a', 'c', 't', 2} // RPCRenewContract is the specifier to renewing an existing contract. RPCRenewContract = types.Specifier{'R', 'e', 'n', 'e', 'w', 'C', 'o', 'n', 't', 'r', 'a', 'c', 't', 2} // RPCReviseContract is the specifier for revising an existing file // contract. RPCReviseContract = types.Specifier{'R', 'e', 'v', 'i', 's', 'e', 'C', 'o', 'n', 't', 'r', 'a', 'c', 't', 2} // RPCRecentRevision is the specifier for getting the most recent file // contract revision for a given file contract. RPCRecentRevision = types.Specifier{'R', 'e', 'c', 'e', 'n', 't', 'R', 'e', 'v', 'i', 's', 'i', 'o', 'n', 2} // RPCSettings is the specifier for requesting settings from the host. RPCSettings = types.Specifier{'S', 'e', 't', 't', 'i', 'n', 'g', 's', 2} // SectorSize defines how large a sector should be in bytes. The sector // size needs to be a power of two to be compatible with package // merkletree. 4MB has been chosen for the live network because large // sectors significantly reduce the tracking overhead experienced by the // renter and the host. SectorSize = func() uint64 { if build.Release == "dev" { return 1 << 18 // 256 KiB } if build.Release == "standard" { return 1 << 22 // 4 MiB } if build.Release == "testing" { return 1 << 12 // 4 KiB } panic("unrecognized release constant in host - sectorSize") }() ) type ( // A DownloadAction is a description of a download that the renter would // like to make. The MerkleRoot indicates the root of the sector, the // offset indicates what portion of the sector is being downloaded, and the // length indicates how many bytes should be grabbed starting from the // offset. DownloadAction struct { MerkleRoot crypto.Hash Offset uint64 Length uint64 } // HostAnnouncement is an announcement by the host that appears in the // blockchain. 'Specifier' is always 'PrefixHostAnnouncement'. The // announcement is always followed by a signature from the public key of // the whole announcement. HostAnnouncement struct { Specifier types.Specifier NetAddress NetAddress PublicKey types.SiaPublicKey } // HostExternalSettings are the parameters advertised by the host. These // are the values that the renter will request from the host in order to // build its database. HostExternalSettings struct { // MaxBatchSize indicates the maximum size in bytes that a batch is // allowed to be. A batch is an array of revision actions; each // revision action can have a different number of bytes, depending on // the action, so the number of revision actions allowed depends on the // sizes of each. AcceptingContracts bool `json:"acceptingcontracts"` MaxDownloadBatchSize uint64 `json:"maxdownloadbatchsize"` MaxDuration types.BlockHeight `json:"maxduration"` MaxReviseBatchSize uint64 `json:"maxrevisebatchsize"` NetAddress NetAddress `json:"netaddress"` RemainingStorage uint64 `json:"remainingstorage"` SectorSize uint64 `json:"sectorsize"` TotalStorage uint64 `json:"totalstorage"` UnlockHash types.UnlockHash `json:"unlockhash"` WindowSize types.BlockHeight `json:"windowsize"` // Collateral is the amount of collateral that the host will put up for // storage in 'bytes per block', as an assurance to the renter that the // host really is committed to keeping the file. But, because the file // contract is created with no data available, this does leave the host // exposed to an attack by a wealthy renter whereby the renter causes // the host to lockup in-advance a bunch of funds that the renter then // never uses, meaning the host will not have collateral for other // clients. // // MaxCollateral indicates the maximum number of coins that a host is // willing to put into a file contract. Collateral types.Currency `json:"collateral"` MaxCollateral types.Currency `json:"maxcollateral"` // ContractPrice is the number of coins that the renter needs to pay to // the host just to open a file contract with them. Generally, the // price is only to cover the siacoin fees that the host will suffer // when submitting the file contract revision and storage proof to the // blockchain. // // The storage price is the cost per-byte-per-block in hastings of // storing data on the host. // // 'Download' bandwidth price is the cost per byte of downloading data // from the host. // // 'Upload' bandwidth price is the cost per byte of uploading data to // the host. ContractPrice types.Currency `json:"contractprice"` DownloadBandwidthPrice types.Currency `json:"downloadbandwidthprice"` StoragePrice types.Currency `json:"storageprice"` UploadBandwidthPrice types.Currency `json:"uploadbandwidthprice"` // Because the host has a public key, and settings are signed, and // because settings may be MITM'd, settings need a revision number so // that a renter can compare multiple sets of settings and determine // which is the most recent. RevisionNumber uint64 `json:"revisionnumber"` Version string `json:"version"` } // A RevisionAction is a description of an edit to be performed on a file // contract. Three types are allowed, 'ActionDelete', 'ActionInsert', and // 'ActionModify'. ActionDelete just takes a sector index, indicating which // sector is going to be deleted. ActionInsert takes a sector index, and a // full sector of data, indicating that a sector at the index should be // inserted with the provided data. 'Modify' revises the sector at the // given index, rewriting it with the provided data starting from the // 'offset' within the sector. // // Modify could be simulated with an insert and a delete, however an insert // requires a full sector to be uploaded, and a modify can be just a few // kb, which can be significantly faster. RevisionAction struct { Type types.Specifier SectorIndex uint64 Offset uint64 Data []byte } ) // ReadNegotiationAcceptance reads an accept/reject response from r (usually a // net.Conn). If the response is not AcceptResponse, ReadNegotiationAcceptance // returns the response as an error. If the response is StopResponse, // ErrStopResponse is returned, allowing for direct error comparison. // // Note that since errors returned by ReadNegotiationAcceptance are newly // allocated, they cannot be compared to other errors in the traditional // fashion. func ReadNegotiationAcceptance(r io.Reader) error { var resp string err := encoding.ReadObject(r, &resp, NegotiateMaxErrorSize) if err != nil { return err } switch resp { case AcceptResponse: return nil case StopResponse: return ErrStopResponse default: return errors.New(resp) } } // WriteNegotiationAcceptance writes the 'accept' response to w (usually a // net.Conn). func WriteNegotiationAcceptance(w io.Writer) error { return encoding.WriteObject(w, AcceptResponse) } // WriteNegotiationRejection will write a rejection response to w (usually a // net.Conn) and return the input error. If the write fails, the write error // is joined with the input error. func WriteNegotiationRejection(w io.Writer, err error) error { writeErr := encoding.WriteObject(w, err.Error()) if writeErr != nil { return build.JoinErrors([]error{err, writeErr}, "; ") } return err } // WriteNegotiationStop writes the 'stop' response to w (usually a // net.Conn). func WriteNegotiationStop(w io.Writer) error { return encoding.WriteObject(w, StopResponse) } // CreateAnnouncement will take a host announcement and encode it, returning // the exact []byte that should be added to the arbitrary data of a // transaction. func CreateAnnouncement(addr NetAddress, pk types.SiaPublicKey, sk crypto.SecretKey) (signedAnnouncement []byte, err error) { if err := addr.IsValid(); err != nil { return nil, err } // Create the HostAnnouncement and marshal it. annBytes := encoding.Marshal(HostAnnouncement{ Specifier: PrefixHostAnnouncement, NetAddress: addr, PublicKey: pk, }) // Create a signature for the announcement. annHash := crypto.HashBytes(annBytes) sig, err := crypto.SignHash(annHash, sk) if err != nil { return nil, err } // Return the signed announcement. return append(annBytes, sig[:]...), nil } // DecodeAnnouncement decodes announcement bytes into a host announcement, // verifying the prefix and the signature. func DecodeAnnouncement(fullAnnouncement []byte) (na NetAddress, spk types.SiaPublicKey, err error) { // Read the first part of the announcement to get the intended host // announcement. var ha HostAnnouncement dec := encoding.NewDecoder(bytes.NewReader(fullAnnouncement)) err = dec.Decode(&ha) if err != nil { return "", types.SiaPublicKey{}, err } // Check that the announcement was registered as a host announcement. if ha.Specifier != PrefixHostAnnouncement { return "", types.SiaPublicKey{}, ErrAnnNotAnnouncement } // Check that the public key is a recognized type of public key. if ha.PublicKey.Algorithm != types.SignatureEd25519 { return "", types.SiaPublicKey{}, ErrAnnUnrecognizedSignature } // Read the signature out of the reader. var sig crypto.Signature err = dec.Decode(&sig) if err != nil { return "", types.SiaPublicKey{}, err } // Verify the signature. var pk crypto.PublicKey copy(pk[:], ha.PublicKey.Key) annHash := crypto.HashObject(ha) err = crypto.VerifyHash(annHash, pk, sig) if err != nil { return "", types.SiaPublicKey{}, err } return ha.NetAddress, ha.PublicKey, nil } // VerifyFileContractRevisionTransactionSignatures checks that the signatures // on a file contract revision are valid and cover the right fields. func VerifyFileContractRevisionTransactionSignatures(fcr types.FileContractRevision, tsigs []types.TransactionSignature, height types.BlockHeight) error { if len(tsigs) != 2 { return ErrRevisionSigCount } for _, tsig := range tsigs { // The transaction needs to be malleable so that miner fees can be // added. If the whole transaction is covered, it is doomed to have no // fees. if tsig.CoveredFields.WholeTransaction { return ErrRevisionCoveredFields } } txn := types.Transaction{ FileContractRevisions: []types.FileContractRevision{fcr}, TransactionSignatures: tsigs, } // Check that the signatures verify. This will also check that the covered // fields object is not over-aggressive, because if the object is pointing // to elements that haven't been added to the transaction, verification // will fail. return txn.StandaloneValid(height) } Sia-1.0.4/modules/negotiate_test.go000066400000000000000000000064071302033070500172450ustar00rootroot00000000000000package modules import ( "bytes" "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/types" ) // TestAnnouncementHandling checks that CreateAnnouncement and // DecodeAnnouncement work together correctly. func TestAnnouncementHandling(t *testing.T) { t.Parallel() // Create the keys that will be used to generate the announcement. sk, pk, err := crypto.GenerateKeyPair() if err != nil { t.Fatal(err) } spk := types.SiaPublicKey{ Algorithm: types.SignatureEd25519, Key: pk[:], } addr := NetAddress("f.o:1234") // Generate the announcement. annBytes, err := CreateAnnouncement(addr, spk, sk) if err != nil { t.Fatal(err) } // Decode the announcement decAddr, decPubKey, err := DecodeAnnouncement(annBytes) if err != nil { t.Fatal(err) } if decPubKey.Algorithm != spk.Algorithm { t.Error("decoded announcement has the wrong algorithm on the public key") } if decAddr != addr { t.Error("decoded announcement has the wrong net address") } if !bytes.Equal(decPubKey.Key, spk.Key) { t.Error("decoded announcement has the wrong public key") } // Corrupt the data, and see that decoding fails. Decoding should fail // because the signature should not be valid anymore. // // First 16 bytes are the host announcement prefix, followed by 8 bytes // describing the length of the net address, followed by the net address. // Corrupt the net address. annBytes[25]++ _, _, err = DecodeAnnouncement(annBytes) if err != crypto.ErrInvalidSignature { t.Error(err) } annBytes[25]-- // The final byte is going to be a part of the signature. Corrupt the final // byte and verify that there's an error. lastIndex := len(annBytes) - 1 annBytes[lastIndex]++ _, _, err = DecodeAnnouncement(annBytes) if err != crypto.ErrInvalidSignature { t.Error(err) } annBytes[lastIndex]-- // Pass in a bad specifier - change the host announcement type. annBytes[0]++ _, _, err = DecodeAnnouncement(annBytes) if err != ErrAnnNotAnnouncement { t.Error(err) } annBytes[0]-- // Pass in a bad signature algorithm. 16 bytes to pass the specifier, 8+8 bytes to pass the net address. annBytes[33]++ _, _, err = DecodeAnnouncement(annBytes) if err != ErrAnnUnrecognizedSignature { t.Error(err) } annBytes[33]-- // Cause the decoding to fail altogether. _, _, err = DecodeAnnouncement(annBytes[:12]) if err == nil { t.Error(err) } } // TestNegotiationResponses tests the WriteNegotiationAcceptance, // WriteNegotiationRejection, and ReadNegotiationAcceptance functions. func TestNegotiationResponses(t *testing.T) { // Write/Read acceptance buf := new(bytes.Buffer) err := WriteNegotiationAcceptance(buf) if err != nil { t.Fatal(err) } err = ReadNegotiationAcceptance(buf) if err != nil { t.Fatal(err) } // Write/Read rejection buf.Reset() err = WriteNegotiationRejection(buf, ErrLowBalance) if err != ErrLowBalance { t.Fatal(err) } err = ReadNegotiationAcceptance(buf) // can't compare to ErrLowBalance directly; contents are the same, but pointer is different if err == nil || err.Error() != ErrLowBalance.Error() { t.Fatal(err) } // Write/Read StopResponse buf.Reset() err = WriteNegotiationStop(buf) if err != nil { t.Fatal(err) } err = ReadNegotiationAcceptance(buf) if err != ErrStopResponse { t.Fatal(err) } } Sia-1.0.4/modules/netaddress.go000066400000000000000000000120561302033070500163600ustar00rootroot00000000000000package modules import ( "errors" "net" "strconv" "strings" "github.com/NebulousLabs/Sia/build" ) // MaxEncodedNetAddressLength is the maximum length of a NetAddress encoded // with the encode package. 266 was chosen because the maximum length for the // hostname is 254 + 1 for the separating colon + 5 for the port + 8 byte // string length prefix. const MaxEncodedNetAddressLength = 266 // A NetAddress contains the information needed to contact a peer. type NetAddress string // Host removes the port from a NetAddress, returning just the host. If the // address is not of the form "host:port" the empty string is returned. The // port will still be returned for invalid NetAddresses (e.g. "unqualified:0" // will return "unqualified"), but in general you should only call Host on // valid addresses. func (na NetAddress) Host() string { host, _, err := net.SplitHostPort(string(na)) // 'host' is not always the empty string if an error is returned. if err != nil { return "" } return host } // Port returns the NetAddress object's port number. If the address is not of // the form "host:port" the empty string is returned. The port will still be // returned for invalid NetAddresses (e.g. "localhost:0" will return "0"), but // in general you should only call Port on valid addresses. func (na NetAddress) Port() string { _, port, err := net.SplitHostPort(string(na)) // 'port' will not always be the empty string if an error is returned. if err != nil { return "" } return port } // IsLoopback returns true for IP addresses that are on the same machine. func (na NetAddress) IsLoopback() bool { host, _, err := net.SplitHostPort(string(na)) if err != nil { return false } if host == "localhost" { return true } if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() { return true } return false } // IsLocal returns true if the input IP address belongs to a local address // range such as 192.168.x.x or 127.x.x.x func (na NetAddress) IsLocal() bool { // Loopback counts as private. if na.IsLoopback() { return true } // Grab the IP address of the net address. If there is an error parsing, // return false, as it's not a private ip address range. ip := net.ParseIP(na.Host()) if ip == nil { return false } // Determine whether or not the ip is in a CIDR that is considered to be // local. localCIDRs := []string{ "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8", } for _, cidr := range localCIDRs { _, ipnet, _ := net.ParseCIDR(cidr) if ipnet.Contains(ip) { return true } } return false } // IsValid is an extension to IsStdValid that also forbids the loopback // address. IsValid is being phased out in favor of allowing the loopback // address but verifying through other means that the connection is not to // yourself (which is the original reason that the loopback address was // banned). func (na NetAddress) IsValid() error { // Check the loopback address. if na.IsLoopback() && build.Release != "testing" { return errors.New("host is a loopback address") } return na.IsStdValid() } // IsStdValid returns an error if the NetAddress is invalid. A valid NetAddress // is of the form "host:port", such that "host" is either a valid IPv4/IPv6 // address or a valid hostname, and "port" is an integer in the range // [1,65535]. Valid IPv4 addresses, IPv6 addresses, and hostnames are detailed // in RFCs 791, 2460, and 952, respectively. func (na NetAddress) IsStdValid() error { // Verify the port number. host, port, err := net.SplitHostPort(string(na)) if err != nil { return err } portInt, err := strconv.Atoi(port) if err != nil { return errors.New("port is not an integer") } else if portInt < 1 || portInt > 65535 { return errors.New("port is invalid") } // Loopback addresses don't always pass the requirements below, and // therefore must be checked separately. if na.IsLoopback() { return nil } // First try to parse host as an IP address; if that fails, assume it is a // hostname. if ip := net.ParseIP(host); ip != nil { if ip.IsUnspecified() { return errors.New("host is the unspecified address") } } else { // Hostnames can have a trailing dot (which indicates that the hostname is // fully qualified), but we ignore it for validation purposes. if strings.HasSuffix(host, ".") { host = host[:len(host)-1] } if len(host) < 1 || len(host) > 253 { return errors.New("invalid hostname length") } labels := strings.Split(host, ".") if len(labels) == 1 { return errors.New("unqualified hostname") } for _, label := range labels { if len(label) < 1 || len(label) > 63 { return errors.New("hostname contains label with invalid length") } if strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") { return errors.New("hostname contains label that starts or ends with a hyphen") } for _, r := range strings.ToLower(label) { isLetter := 'a' <= r && r <= 'z' isNumber := '0' <= r && r <= '9' isHyphen := r == '-' if !(isLetter || isNumber || isHyphen) { return errors.New("host contains invalid characters") } } } } return nil } Sia-1.0.4/modules/netaddress_test.go000066400000000000000000000215121302033070500174140ustar00rootroot00000000000000package modules import ( "net" "strings" "testing" ) var ( // Networks such as 10.0.0.x have been omitted from testing - behavior // for these networks is currently undefined. invalidAddrs = []string{ // Garbage addresses "", "foo:bar:baz", "garbage:6146:616", // Missing host / port ":", "111.111.111.111", "12.34.45.64", "[::2]", "::2", "foo", "hn.com", "世界", "foo:", "世界:", ":foo", ":世界", "localhost:", "[::1]:", // Invalid host / port chars "localhost:-", "[::1]:-", "foo:{}", "{}:123", " foo:123", "foo :123", "f oo:123", "foo: 123", "foo:123 ", "foo:1 23", "\x00:123", "foo:\x00", "世界:123", "bar:世界", "世:界", `":"`, // Unspecified address "[::]:bar", "0.0.0.0:bar", // Invalid hostnames "unqualifiedhost:123", "Yo-Amazon.we-are-really-happy-for-you.and-we-will-let-you-finish.but-Sia-is-the-best-cloud-storage-of-all-time.of-all-time-of-all-time-of-all-time-of-all-time-of-all-time.of-all-time-of-all-time-of-all-time-of-all-time-of-all-time.of-all-time-of-all-time:123", strings.Repeat("a", 64) + ".com:123", // 64 char long label too long. strings.Repeat(strings.Repeat("a", 62)+".", 4) + "co:123", // 254 char long hostname too long. strings.Repeat(strings.Repeat("a", 62)+".", 4) + "co.:123", // 254 char long hostname with trailing dot too long. "-foo.bar:123", "foo-.bar:123", "foo.-bar:123", "foo.bar-:123", "foo-bar.-baz:123", "foo-bar.baz-:123", "foo.-bar.baz:123", "foo.bar-.baz:123", ".:123", ".foo.com:123", "foo.com..:123", // invalid port numbers "foo:0", "foo:65536", "foo:-100", "foo:1000000", "localhost:0", "[::1]:0", } validAddrs = []string{ // Loopback address (valid in testing only, can't really test this well) "localhost:123", "127.0.0.1:123", "[::1]:123", // Valid addresses. "foo.com:1", "foo.com.:1", "a.b.c:1", "a.b.c.:1", "foo-bar.com:123", "FOO.com:1", "1foo.com:1", "tld.foo.com:1", "hn.com:8811", strings.Repeat("foo.", 63) + "f:123", // 253 chars long strings.Repeat("foo.", 63) + "f.:123", // 254 chars long, 253 chars long without trailing dot strings.Repeat(strings.Repeat("a", 63)+".", 3) + "a:123", // 3x63 char length labels + 1x1 char length label without trailing dot strings.Repeat(strings.Repeat("a", 63)+".", 3) + ":123", // 3x63 char length labels with trailing dot "[::2]:65535", "111.111.111.111:111", "12.34.45.64:7777", } ) // TestHostPort tests the Host and Port methods of the NetAddress type. func TestHostPort(t *testing.T) { t.Parallel() // Test valid addrs. for _, addr := range validAddrs { na := NetAddress(addr) host := na.Host() port := na.Port() expectedHost, expectedPort, err := net.SplitHostPort(addr) if err != nil { t.Fatal(err) } if host != expectedHost { t.Errorf("Host() returned unexpected host for NetAddress '%v': expected '%v', got '%v'", na, expectedHost, host) } if port != expectedPort { t.Errorf("Port() returned unexpected port for NetAddress '%v': expected '%v', got '%v'", na, expectedPort, port) } } // Test that Host / Port return "" when net.SplitHostPort errors na := NetAddress("::") host := na.Host() port := na.Port() if host != "" { t.Error("expected Host() to return blank for an un-splittable NetAddress, but it returned:", host) } if port != "" { t.Error("expected Port() to return blank for an un-splittable NetAddress, but it returned:", port) } } // TestIsLoopback tests the IsLoopback method of the NetAddress type. func TestIsLoopback(t *testing.T) { t.Parallel() testSet := []struct { query NetAddress desiredResponse bool }{ // Localhost tests. {"localhost", false}, {"localhost:1234", true}, {"127.0.0.1", false}, {"127.0.0.1:6723", true}, {"::1", false}, {"[::1]:7124", true}, // Local network tests. {"10.0.0.0", false}, {"10.0.0.0:1234", false}, {"10.2.2.5", false}, {"10.2.2.5:16432", false}, {"10.255.255.255", false}, {"10.255.255.255:16432", false}, {"172.16.0.0", false}, {"172.16.0.0:1234", false}, {"172.26.2.5", false}, {"172.26.2.5:16432", false}, {"172.31.255.255", false}, {"172.31.255.255:16432", false}, {"192.168.0.0", false}, {"192.168.0.0:1234", false}, {"192.168.2.5", false}, {"192.168.2.5:16432", false}, {"192.168.255.255", false}, {"192.168.255.255:16432", false}, {"1234:0000:0000:0000:0000:0000:0000:0000", false}, {"[1234:0000:0000:0000:0000:0000:0000:0000]:1234", false}, {"fc00:0000:0000:0000:0000:0000:0000:0000", false}, {"[fc00:0000:0000:0000:0000:0000:0000:0000]:1234", false}, {"fd00:0000:0000:0000:0000:0000:0000:0000", false}, {"[fd00:0000:0000:0000:0000:0000:0000:0000]:1234", false}, {"fd30:0000:0000:0000:0000:0000:0000:0000", false}, {"[fd30:0000:0000:0000:0000:0000:0000:0000]:1234", false}, {"fd00:0000:0030:0000:0000:0000:0000:0000", false}, {"[fd00:0000:0030:0000:0000:0000:0000:0000]:1234", false}, {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false}, {"[fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]:1234", false}, {"fe00:0000:0000:0000:0000:0000:0000:0000", false}, {"[fe00:0000:0000:0000:0000:0000:0000:0000]:1234", false}, // Unspecified address tests. {"0.0.0.0:1234", false}, {"[::]:1234", false}, // Public name tests. {"hn.com", false}, {"hn.com:8811", false}, {"2.34.45.64", false}, {"2.34.45.64:7777", false}, {"12.34.45.64", false}, {"12.34.45.64:7777", false}, {"122.34.45.64", false}, {"122.34.45.64:7777", false}, {"197.34.45.64", false}, {"197.34.45.64:7777", false}, {"222.34.45.64", false}, {"222.34.45.64:7777", false}, // Garbage name tests. {"", false}, {"garbage", false}, {"garbage:6432", false}, {"garbage:6146:616", false}, {"::1:4646", false}, {"[::1]", false}, } for _, test := range testSet { if test.query.IsLoopback() != test.desiredResponse { t.Error("test failed:", test, test.query.IsLoopback()) } } } // TestIsValid tests that IsValid only returns nil for valid addresses. func TestIsValid(t *testing.T) { t.Parallel() for _, addr := range validAddrs { na := NetAddress(addr) if err := na.IsValid(); err != nil { t.Errorf("IsValid returned non-nil for valid NetAddress %q: %v", addr, err) } } for _, addr := range invalidAddrs { na := NetAddress(addr) if err := na.IsValid(); err == nil { t.Errorf("IsValid returned nil for an invalid NetAddress %q: %v", addr, err) } } } // TestIsLocal checks that the correct values are returned for all local IP // addresses. func TestIsLocal(t *testing.T) { t.Parallel() testSet := []struct { query NetAddress desiredResponse bool }{ // Localhost tests. {"localhost", false}, {"localhost:1234", true}, {"127.0.0.1", false}, {"127.0.0.1:6723", true}, {"::1", false}, {"[::1]:7124", true}, // Local network tests. {"10.0.0.0", false}, {"10.0.0.0:1234", true}, {"10.2.2.5", false}, {"10.2.2.5:16432", true}, {"10.255.255.255", false}, {"10.255.255.255:16432", true}, {"172.16.0.0", false}, {"172.16.0.0:1234", true}, {"172.26.2.5", false}, {"172.26.2.5:16432", true}, {"172.31.255.255", false}, {"172.31.255.255:16432", true}, {"192.168.0.0", false}, {"192.168.0.0:1234", true}, {"192.168.2.5", false}, {"192.168.2.5:16432", true}, {"192.168.255.255", false}, {"192.168.255.255:16432", true}, {"1234:0000:0000:0000:0000:0000:0000:0000", false}, {"[1234:0000:0000:0000:0000:0000:0000:0000]:1234", false}, {"fc00:0000:0000:0000:0000:0000:0000:0000", false}, {"[fc00:0000:0000:0000:0000:0000:0000:0000]:1234", false}, {"fd00:0000:0000:0000:0000:0000:0000:0000", false}, {"[fd00:0000:0000:0000:0000:0000:0000:0000]:1234", true}, {"fd30:0000:0000:0000:0000:0000:0000:0000", false}, {"[fd30:0000:0000:0000:0000:0000:0000:0000]:1234", true}, {"fd00:0000:0030:0000:0000:0000:0000:0000", false}, {"[fd00:0000:0030:0000:0000:0000:0000:0000]:1234", true}, {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false}, {"[fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]:1234", true}, {"fe00:0000:0000:0000:0000:0000:0000:0000", false}, {"[fe00:0000:0000:0000:0000:0000:0000:0000]:1234", false}, // Unspecified address tests. {"0.0.0.0:1234", false}, {"[::]:1234", false}, // Public name tests. {"hn.com", false}, {"hn.com:8811", false}, {"2.34.45.64", false}, {"2.34.45.64:7777", false}, {"12.34.45.64", false}, {"12.34.45.64:7777", false}, {"122.34.45.64", false}, {"122.34.45.64:7777", false}, {"197.34.45.64", false}, {"197.34.45.64:7777", false}, {"222.34.45.64", false}, {"222.34.45.64:7777", false}, // Garbage name tests. {"", false}, {"garbage", false}, {"garbage:6432", false}, {"garbage:6146:616", false}, {"::1:4646", false}, {"[::1]", false}, } for _, test := range testSet { if test.query.IsLocal() != test.desiredResponse { t.Error("test failed:", test, test.query.IsLocal()) } } } Sia-1.0.4/modules/renter.go000066400000000000000000000143271302033070500155260ustar00rootroot00000000000000package modules import ( "io" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/types" ) const ( // RenterDir is the name of the directory that is used to store the // renter's persistent data. RenterDir = "renter" ) // An ErasureCoder is an error-correcting encoder and decoder. type ErasureCoder interface { // NumPieces is the number of pieces returned by Encode. NumPieces() int // MinPieces is the minimum number of pieces that must be present to // recover the original data. MinPieces() int // Encode splits data into equal-length pieces, with some pieces // containing parity data. Encode(data []byte) ([][]byte, error) // Recover recovers the original data from pieces (including parity) and // writes it to w. pieces should be identical to the slice returned by // Encode (length and order must be preserved), but with missing elements // set to nil. n is the number of bytes to be written to w; this is // necessary because pieces may have been padded with zeros during // encoding. Recover(pieces [][]byte, n uint64, w io.Writer) error } // FileUploadParams contains the information used by the Renter to upload a // file. type FileUploadParams struct { Source string SiaPath string ErasureCode ErasureCoder } // FileInfo provides information about a file. type FileInfo struct { SiaPath string `json:"siapath"` Filesize uint64 `json:"filesize"` Available bool `json:"available"` Renewing bool `json:"renewing"` Redundancy float64 `json:"redundancy"` UploadProgress float64 `json:"uploadprogress"` Expiration types.BlockHeight `json:"expiration"` } // DownloadInfo provides information about a file that has been requested for // download. type DownloadInfo struct { SiaPath string `json:"siapath"` Destination string `json:"destination"` Filesize uint64 `json:"filesize"` Received uint64 `json:"received"` StartTime time.Time `json:"starttime"` } // An Allowance dictates how much the Renter is allowed to spend in a given // period. Note that funds are spent on both storage and bandwidth. type Allowance struct { Funds types.Currency `json:"funds"` Hosts uint64 `json:"hosts"` Period types.BlockHeight `json:"period"` RenewWindow types.BlockHeight `json:"renewwindow"` } // RenterSettings control the behavior of the Renter. type RenterSettings struct { Allowance Allowance `json:"allowance"` } // RenterFinancialMetrics contains metrics about how much the Renter has // spent on storage, uploads, and downloads. type RenterFinancialMetrics struct { // ContractSpending is how much the Renter has paid into file contracts // formed with hosts. Note that some of this money may be returned to the // Renter when the contract ends. To calculate how much will be returned, // subtract the storage, upload, and download metrics from // ContractSpending. ContractSpending types.Currency `json:"contractspending"` DownloadSpending types.Currency `json:"downloadspending"` StorageSpending types.Currency `json:"storagespending"` UploadSpending types.Currency `json:"uploadspending"` } // A HostDBEntry represents one host entry in the Renter's host DB. It // aggregates the host's external settings with its public key. type HostDBEntry struct { HostExternalSettings PublicKey types.SiaPublicKey `json:"publickey"` } // A RenterContract contains all the metadata necessary to revise or renew a // file contract. type RenterContract struct { FileContract types.FileContract `json:"filecontract"` ID types.FileContractID `json:"id"` LastRevision types.FileContractRevision `json:"lastrevision"` LastRevisionTxn types.Transaction `json:"lastrevisiontxn"` MerkleRoots []crypto.Hash `json:"merkleroots"` NetAddress NetAddress `json:"netaddress"` SecretKey crypto.SecretKey `json:"secretkey"` } // EndHeight returns the height at which the host is no longer obligated to // store contract data. func (rc *RenterContract) EndHeight() types.BlockHeight { return rc.LastRevision.NewWindowStart } // RenterFunds returns the funds remaining in the contract's Renter payout as // of the most recent revision. func (rc *RenterContract) RenterFunds() types.Currency { return rc.LastRevision.NewValidProofOutputs[0].Value } // A Renter uploads, tracks, repairs, and downloads a set of files for the // user. type Renter interface { // ActiveHosts provides the list of hosts that the renter is selecting, // sorted by preference. ActiveHosts() []HostDBEntry // AllHosts returns the full list of hosts known to the renter. AllHosts() []HostDBEntry // Close closes the Renter. Close() error // Contracts returns the contracts formed by the renter. Contracts() []RenterContract // DeleteFile deletes a file entry from the renter. DeleteFile(path string) error // Download downloads a file to the given destination. Download(path, destination string) error // DownloadQueue lists all the files that have been scheduled for download. DownloadQueue() []DownloadInfo // FileList returns information on all of the files stored by the renter. FileList() []FileInfo // FinancialMetrics returns the financial metrics of the Renter. FinancialMetrics() RenterFinancialMetrics // LoadSharedFiles loads a '.sia' file into the renter. A .sia file may // contain multiple files. The paths of the added files are returned. LoadSharedFiles(source string) ([]string, error) // LoadSharedFilesAscii loads an ASCII-encoded '.sia' file into the // renter. LoadSharedFilesAscii(asciiSia string) ([]string, error) // RenameFile changes the path of a file. RenameFile(path, newPath string) error // Settings returns the Renter's current settings. Settings() RenterSettings // SetSettings sets the Renter's settings. SetSettings(RenterSettings) error // ShareFiles creates a '.sia' file that can be shared with others. ShareFiles(paths []string, shareDest string) error // ShareFilesAscii creates an ASCII-encoded '.sia' file. ShareFilesAscii(paths []string) (asciiSia string, err error) // Upload uploads a file using the input parameters. Upload(FileUploadParams) error } Sia-1.0.4/modules/renter/000077500000000000000000000000001302033070500151705ustar00rootroot00000000000000Sia-1.0.4/modules/renter/contractor/000077500000000000000000000000001302033070500173465ustar00rootroot00000000000000Sia-1.0.4/modules/renter/contractor/allowance.go000066400000000000000000000137621302033070500216530ustar00rootroot00000000000000package contractor import ( "errors" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( errAllowanceNoHosts = errors.New("hosts must be non-zero") errAllowanceZeroPeriod = errors.New("period must be non-zero") errAllowanceWindowSize = errors.New("renew window must be less than period") // ErrAllowanceZeroWindow is returned when the caller requests a // zero-length renewal window. This will happen if the caller sets the // period to 1 block, since RenewWindow := period / 2. ErrAllowanceZeroWindow = errors.New("renew window must be non-zero") ) // contractEndHeight returns the height at which the Contractor's contracts // end. If there are no contracts, it returns zero. func (c *Contractor) contractEndHeight() types.BlockHeight { var endHeight types.BlockHeight for _, contract := range c.contracts { endHeight = contract.EndHeight() break } return endHeight } // SetAllowance sets the amount of money the Contractor is allowed to spend on // contracts over a given time period, divided among the number of hosts // specified. Note that Contractor can start forming contracts as soon as // SetAllowance is called; that is, it may block. // // In most cases, SetAllowance will renew existing contracts instead of // forming new ones. This preserves the data on those hosts. When this occurs, // the renewed contracts will atomically replace their previous versions. If // SetAllowance is interrupted, renewed contracts may be lost, though the // allocated funds will eventually be returned, // // TODO: can an Editor or Downloader be used across renewals? // TODO: will hosts allow renewing the same contract twice? // // NOTE: At this time, transaction fees are not counted towards the allowance. // This means the contractor may spend more than allowance.Funds. func (c *Contractor) SetAllowance(a modules.Allowance) error { // sanity checks if a.Hosts == 0 { return errAllowanceNoHosts } else if a.Period == 0 { return errAllowanceZeroPeriod } else if a.RenewWindow == 0 { return ErrAllowanceZeroWindow } else if a.RenewWindow >= a.Period { return errAllowanceWindowSize } // check that allowance is sufficient to store at least one sector numSectors, err := maxSectors(a, c.hdb, c.tpool) if err != nil { return err } else if numSectors == 0 { return ErrInsufficientAllowance } c.mu.RLock() shouldRenew := a.Period != c.allowance.Period || a.Funds.Cmp(c.allowance.Funds) != 0 shouldWait := c.blockHeight+a.Period < c.contractEndHeight() remaining := int(a.Hosts) - len(c.contracts) c.mu.RUnlock() if !shouldRenew { // If no contracts need renewing, just form new contracts. return c.managedFormAllowanceContracts(remaining, numSectors, a) } else if shouldWait { // If the new period would result in an earlier endHeight, we can't // renew; instead, set the allowance without modifying any contracts. // They will be renewed with the new allowance when they expire. c.mu.Lock() c.allowance = a err = c.saveSync() c.mu.Unlock() return err } c.mu.RLock() // gather contracts to renew var renewSet []modules.RenterContract for _, contract := range c.contracts { renewSet = append(renewSet, contract) } // calculate new endHeight; if the period has not changed, the endHeight // should not change either endHeight := c.blockHeight + a.Period if a.Period == c.allowance.Period && len(c.contracts) > 0 { // COMPAT v0.6.0 - old hosts require end height increase by at least 1 endHeight = c.contractEndHeight() + 1 } c.mu.RUnlock() // renew existing contracts with new allowance parameters newContracts := make(map[types.FileContractID]modules.RenterContract) for _, contract := range renewSet { newContract, err := c.managedRenew(contract, numSectors, endHeight) if err != nil { c.log.Printf("WARN: failed to renew contract with %v; a new contract will be formed in its place", contract.NetAddress) remaining++ continue } newContracts[newContract.ID] = newContract if len(newContracts) >= int(a.Hosts) { break } if build.Release != "testing" { // sleep for 1 minute to alleviate potential block propagation issues time.Sleep(60 * time.Second) } } // if we did not renew enough contracts, form new ones if remaining > 0 { formed, err := c.managedFormContracts(remaining, numSectors, endHeight) if err != nil { return err } for _, contract := range formed { newContracts[contract.ID] = contract } } // if we weren't able to form anything, return an error if len(newContracts) == 0 { return errors.New("unable to form or renew any contracts") } // Set the allowance and replace the contract set c.mu.Lock() c.allowance = a c.contracts = newContracts // update metrics var spending types.Currency for _, contract := range c.contracts { spending = spending.Add(contract.RenterFunds()) } c.financialMetrics.ContractSpending = spending err = c.saveSync() c.mu.Unlock() return err } // managedFormAllowanceContracts handles the special case where no contracts // need to be renewed when setting the allowance. func (c *Contractor) managedFormAllowanceContracts(n int, numSectors uint64, a modules.Allowance) error { if n <= 0 { return nil } // if we're forming contracts but not renewing, the new contracts should // have the same endHeight as the existing ones. Otherwise, the endHeight // should be a full period. c.mu.RLock() endHeight := c.blockHeight + a.Period if len(c.contracts) > 0 { endHeight = c.contractEndHeight() } c.mu.RUnlock() // form the contracts formed, err := c.managedFormContracts(n, numSectors, endHeight) if err != nil { return err } // Set the allowance and replace the contract set c.mu.Lock() c.allowance = a for _, contract := range formed { c.contracts[contract.ID] = contract } // update metrics var spending types.Currency for _, contract := range c.contracts { spending = spending.Add(contract.RenterFunds()) } c.financialMetrics.ContractSpending = spending err = c.saveSync() c.mu.Unlock() return err } Sia-1.0.4/modules/renter/contractor/consts.go000066400000000000000000000025101302033070500212040ustar00rootroot00000000000000package contractor import ( "github.com/NebulousLabs/Sia/build" ) const ( // estimatedFileContractTransactionSize provides the estimated size of // the average file contract in bytes. estimatedFileContractTransactionSize = 1200 ) var ( // minHostsForEstimations describes the minimum number of hosts that // are needed to make broad estimations such as the number of sectors // that you can store on the network for a given allowance. minHostsForEstimations = func() int { switch build.Release { case "dev": // The number is set lower than standard so that it can // be reached/exceeded easily within development // environments, but set high enough that it's also // easy to fall short within the development // environments. return 5 case "standard": // Hosts can have a lot of variance. Selecting too many // hosts will high-ball the price estimation, but users // shouldn't be selecting rewer hosts, and if there are // too few hosts being selected for estimation there is // a risk of underestimating the actual price, which is // something we'd rather avoid. return 10 case "testing": // Testing tries to happen as fast as possible, // therefore tends to run with a lot fewer hosts. return 4 default: panic("unrecognized build.Release in minHostsForEstimations") } }() ) Sia-1.0.4/modules/renter/contractor/contractor.go000066400000000000000000000121031302033070500220500ustar00rootroot00000000000000package contractor import ( "errors" "os" "path/filepath" "sync" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" siasync "github.com/NebulousLabs/Sia/sync" "github.com/NebulousLabs/Sia/types" ) var ( errNilCS = errors.New("cannot create contractor with nil consensus set") errNilWallet = errors.New("cannot create contractor with nil wallet") errNilTpool = errors.New("cannot create contractor with nil transaction pool") ) // A cachedRevision contains changes that would be applied to a RenterContract // if a contract revision succeeded. The contractor must cache these changes // as a safeguard against desynchronizing with the host. // TODO: save a diff of the Merkle roots instead of all of them. type cachedRevision struct { revision types.FileContractRevision merkleRoots []crypto.Hash } // A Contractor negotiates, revises, renews, and provides access to file // contracts. type Contractor struct { // dependencies cs consensusSet hdb hostDB log *persist.Logger persist persister tpool transactionPool wallet wallet allowance modules.Allowance blockHeight types.BlockHeight cachedRevisions map[types.FileContractID]cachedRevision contracts map[types.FileContractID]modules.RenterContract downloaders map[types.FileContractID]*hostDownloader editors map[types.FileContractID]*hostEditor lastChange modules.ConsensusChangeID renewedIDs map[types.FileContractID]types.FileContractID renewing map[types.FileContractID]bool // prevent revising during renewal revising map[types.FileContractID]bool // prevent overlapping revisions financialMetrics modules.RenterFinancialMetrics mu sync.RWMutex // in addition to mu, a separate lock enforces that multiple goroutines // won't try to simultaneously edit the contract set. editLock siasync.TryMutex } // Allowance returns the current allowance. func (c *Contractor) Allowance() modules.Allowance { c.mu.RLock() defer c.mu.RUnlock() return c.allowance } // FinancialMetrics returns the financial metrics of the Contractor. func (c *Contractor) FinancialMetrics() modules.RenterFinancialMetrics { c.mu.RLock() defer c.mu.RUnlock() return c.financialMetrics } // Contract returns the latest contract formed with the specified host. func (c *Contractor) Contract(hostAddr modules.NetAddress) (modules.RenterContract, bool) { c.mu.RLock() defer c.mu.RUnlock() for _, c := range c.contracts { if c.NetAddress == hostAddr { return c, true } } return modules.RenterContract{}, false } // Contracts returns the contracts formed by the contractor. func (c *Contractor) Contracts() (cs []modules.RenterContract) { c.mu.RLock() defer c.mu.RUnlock() for _, c := range c.contracts { cs = append(cs, c) } return } // resolveID returns the ID of the most recent renewal of id. func (c *Contractor) resolveID(id types.FileContractID) types.FileContractID { if newID, ok := c.renewedIDs[id]; ok && newID != id { return c.resolveID(newID) } return id } // New returns a new Contractor. func New(cs consensusSet, wallet walletShim, tpool transactionPool, hdb hostDB, persistDir string) (*Contractor, error) { // Check for nil inputs. if cs == nil { return nil, errNilCS } if wallet == nil { return nil, errNilWallet } if tpool == nil { return nil, errNilTpool } // Create the persist directory if it does not yet exist. err := os.MkdirAll(persistDir, 0700) if err != nil { return nil, err } // Create the logger. logger, err := persist.NewFileLogger(filepath.Join(persistDir, "contractor.log")) if err != nil { return nil, err } // Create Contractor using production dependencies. return newContractor(cs, &walletBridge{w: wallet}, tpool, hdb, newPersist(persistDir), logger) } // newContractor creates a Contractor using the provided dependencies. func newContractor(cs consensusSet, w wallet, tp transactionPool, hdb hostDB, p persister, l *persist.Logger) (*Contractor, error) { // Create the Contractor object. c := &Contractor{ cs: cs, hdb: hdb, log: l, persist: p, tpool: tp, wallet: w, cachedRevisions: make(map[types.FileContractID]cachedRevision), contracts: make(map[types.FileContractID]modules.RenterContract), downloaders: make(map[types.FileContractID]*hostDownloader), editors: make(map[types.FileContractID]*hostEditor), renewedIDs: make(map[types.FileContractID]types.FileContractID), renewing: make(map[types.FileContractID]bool), revising: make(map[types.FileContractID]bool), } // Load the prior persistence structures. err := c.load() if err != nil && !os.IsNotExist(err) { return nil, err } err = cs.ConsensusSetSubscribe(c, c.lastChange) if err == modules.ErrInvalidConsensusChangeID { // Reset the contractor consensus variables and try rescanning. c.blockHeight = 0 c.lastChange = modules.ConsensusChangeBeginning err = cs.ConsensusSetSubscribe(c, c.lastChange) } if err != nil { return nil, errors.New("contractor subscription failed: " + err.Error()) } return c, nil } Sia-1.0.4/modules/renter/contractor/contractor_test.go000066400000000000000000000224031302033070500231130ustar00rootroot00000000000000package contractor import ( "encoding/json" "io/ioutil" "os" "path/filepath" "testing" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // newStub is used to test the New function. It implements all of the contractor's // dependencies. type newStub struct{} // consensus set stubs func (newStub) ConsensusSetSubscribe(modules.ConsensusSetSubscriber, modules.ConsensusChangeID) error { return nil } func (newStub) Synced() bool { return true } // wallet stubs func (newStub) NextAddress() (uc types.UnlockConditions, err error) { return } func (newStub) StartTransaction() modules.TransactionBuilder { return nil } // transaction pool stubs func (newStub) AcceptTransactionSet([]types.Transaction) error { return nil } func (newStub) FeeEstimation() (a types.Currency, b types.Currency) { return } // hdb stubs func (newStub) Host(modules.NetAddress) (settings modules.HostDBEntry, ok bool) { return } func (newStub) RandomHosts(int, []modules.NetAddress) []modules.HostDBEntry { return nil } // TestNew tests the New function. func TestNew(t *testing.T) { // Using a stub implementation of the dependencies is fine, as long as its // non-nil. var stub newStub dir := build.TempDir("contractor", "TestNew") // Sane values. _, err := New(stub, stub, stub, stub, dir) if err != nil { t.Fatalf("expected nil, got %v", err) } // Nil consensus set. _, err = New(nil, stub, stub, stub, dir) if err != errNilCS { t.Fatalf("expected %v, got %v", errNilCS, err) } // Nil wallet. _, err = New(stub, nil, stub, stub, dir) if err != errNilWallet { t.Fatalf("expected %v, got %v", errNilWallet, err) } // Nil transaction pool. _, err = New(stub, stub, nil, stub, dir) if err != errNilTpool { t.Fatalf("expected %v, got %v", errNilTpool, err) } // Bad persistDir. _, err = New(stub, stub, stub, stub, "") if !os.IsNotExist(err) { t.Fatalf("expected invalid directory, got %v", err) } // Corrupted persist file. ioutil.WriteFile(filepath.Join(dir, "contractor.json"), []byte{1, 2, 3}, 0666) _, err = New(stub, stub, stub, stub, dir) if _, ok := err.(*json.SyntaxError); !ok { t.Fatalf("expected invalid json, got %v", err) } } // TestContract tests the Contract method. func TestContract(t *testing.T) { c := &Contractor{ contracts: map[types.FileContractID]modules.RenterContract{ {1}: {ID: types.FileContractID{1}, NetAddress: "foo"}, {2}: {ID: types.FileContractID{2}, NetAddress: "bar"}, {3}: {ID: types.FileContractID{3}, NetAddress: "baz"}, }, } tests := []struct { addr modules.NetAddress exists bool contractID types.FileContractID }{ {"foo", true, types.FileContractID{1}}, {"bar", true, types.FileContractID{2}}, {"baz", true, types.FileContractID{3}}, {"quux", false, types.FileContractID{}}, {"nope", false, types.FileContractID{}}, } for _, test := range tests { contract, ok := c.Contract(test.addr) if ok != test.exists { t.Errorf("%v: expected %v, got %v", test.addr, test.exists, ok) } else if contract.ID != test.contractID { t.Errorf("%v: expected %v, got %v", test.addr, test.contractID, contract.ID) } } // delete all contracts c.contracts = map[types.FileContractID]modules.RenterContract{} for _, test := range tests { _, ok := c.Contract(test.addr) if ok { t.Error("no contracts should remain") } } } // TestContracts tests the Contracts method. func TestContracts(t *testing.T) { c := &Contractor{ contracts: map[types.FileContractID]modules.RenterContract{ {1}: {ID: types.FileContractID{1}, NetAddress: "foo"}, {2}: {ID: types.FileContractID{2}, NetAddress: "bar"}, {3}: {ID: types.FileContractID{3}, NetAddress: "baz"}, }, } for _, contract := range c.Contracts() { if exp := c.contracts[contract.ID]; exp.NetAddress != contract.NetAddress { t.Errorf("contract does not match: expected %v, got %v", exp.NetAddress, contract.NetAddress) } } } // TestResolveID tests the resolveID method. func TestResolveID(t *testing.T) { c := &Contractor{ renewedIDs: map[types.FileContractID]types.FileContractID{ {1}: {2}, {2}: {3}, {3}: {4}, {5}: {6}, }, } tests := []struct { id types.FileContractID resolved types.FileContractID }{ {types.FileContractID{0}, types.FileContractID{0}}, {types.FileContractID{1}, types.FileContractID{4}}, {types.FileContractID{2}, types.FileContractID{4}}, {types.FileContractID{3}, types.FileContractID{4}}, {types.FileContractID{4}, types.FileContractID{4}}, {types.FileContractID{5}, types.FileContractID{6}}, } for _, test := range tests { if r := c.resolveID(test.id); r != test.resolved { t.Errorf("expected %v -> %v, got %v", test.id, test.resolved, r) } } } // TestAllowance tests the Allowance method. func TestAllowance(t *testing.T) { c := &Contractor{ allowance: modules.Allowance{ Funds: types.NewCurrency64(1), Period: 2, Hosts: 3, }, } a := c.Allowance() if a.Funds.Cmp(c.allowance.Funds) != 0 || a.Period != c.allowance.Period || a.Hosts != c.allowance.Hosts { t.Fatal("Allowance did not return correct allowance:", a, c.allowance) } } // stubHostDB mocks the hostDB dependency using zero-valued implementations of // its methods. type stubHostDB struct{} func (stubHostDB) Host(modules.NetAddress) (h modules.HostDBEntry, ok bool) { return } func (stubHostDB) RandomHosts(int, []modules.NetAddress) (hs []modules.HostDBEntry) { return } // TestIntegrationSetAllowance tests the SetAllowance method. func TestIntegrationSetAllowance(t *testing.T) { if testing.Short() { t.SkipNow() } // create testing trio h, c, m, err := newTestingTrio("TestIntegrationSetAllowance") if err != nil { t.Fatal(err) } // bad args var a modules.Allowance err = c.SetAllowance(a) if err != errAllowanceNoHosts { t.Errorf("expected %q, got %q", errAllowanceNoHosts, err) } a.Hosts = 1 err = c.SetAllowance(a) if err != errAllowanceZeroPeriod { t.Errorf("expected %q, got %q", errAllowanceZeroPeriod, err) } a.Period = 20 err = c.SetAllowance(a) if err != ErrAllowanceZeroWindow { t.Errorf("expected %q, got %q", ErrAllowanceZeroWindow, err) } a.RenewWindow = 20 err = c.SetAllowance(a) if err != errAllowanceWindowSize { t.Errorf("expected %q, got %q", errAllowanceWindowSize, err) } // reasonable values; should succeed a.Funds = types.SiacoinPrecision.Mul64(100) a.RenewWindow = 10 err = c.SetAllowance(a) if err != nil { t.Fatal(err) } if len(c.contracts) != 1 { t.Error("expected 1 contract, got", len(c.contracts)) } // set same allowance; should no-op err = c.SetAllowance(a) if err != nil { t.Fatal(err) } if len(c.contracts) != 1 { t.Fatal("expected 1 contract, got", len(c.contracts)) } // reannounce host on different IP (easier than creating a new host) addr := modules.NetAddress("127.0.0.1:" + c.Contracts()[0].NetAddress.Port()) err = h.AnnounceAddress(addr) if err != nil { t.Fatal(err) } m.AddBlock() // wait for hostdb to scan host for i := 0; i < 500 && len(c.hdb.RandomHosts(2, nil)) != 2; i++ { time.Sleep(time.Millisecond) } // set allowance with Hosts = 2; should only form one new contract a.Hosts = 2 err = c.SetAllowance(a) if err != nil { t.Fatal(err) } if len(c.contracts) != 2 { t.Fatal("expected 2 contracts, got", len(c.contracts)) } // set allowance with Funds*2; should trigger renewal of both contracts a.Funds = a.Funds.Mul64(2) err = c.SetAllowance(a) if err != nil { t.Fatal(err) } if len(c.contracts) != 2 { t.Fatal("expected 2 contracts, got", len(c.contracts)) } // delete one of the contracts and set allowance with Funds*2; should // trigger 1 renewal and 1 new contract c.mu.Lock() for id, contract := range c.contracts { if contract.NetAddress == addr { delete(c.contracts, id) break } } c.mu.Unlock() a.Funds = a.Funds.Mul64(2) err = c.SetAllowance(a) if err != nil { t.Fatal(err) } if len(c.contracts) != 2 { t.Fatal("expected 2 contracts, got", len(c.contracts)) } // make one of the contracts un-renewable and set allowance with Funds*2; should // trigger 1 renewal failure and 2 new contracts c.mu.Lock() for id, contract := range c.contracts { contract.NetAddress = "foo" c.contracts[id] = contract break } c.mu.Unlock() a.Funds = a.Funds.Mul64(2) err = c.SetAllowance(a) if err != nil { t.Fatal(err) } if len(c.contracts) != 2 { t.Fatal("expected 2 contracts, got", len(c.contracts)) } } // testWalletShim is used to test the walletBridge type. type testWalletShim struct { nextAddressCalled bool startTxnCalled bool } // These stub implementations for the walletShim interface set their respective // booleans to true, allowing tests to verify that they have been called. func (ws *testWalletShim) NextAddress() (types.UnlockConditions, error) { ws.nextAddressCalled = true return types.UnlockConditions{}, nil } func (ws *testWalletShim) StartTransaction() modules.TransactionBuilder { ws.startTxnCalled = true return nil } // TestWalletBridge tests the walletBridge type. func TestWalletBridge(t *testing.T) { shim := new(testWalletShim) bridge := walletBridge{shim} bridge.NextAddress() if !shim.nextAddressCalled { t.Error("NextAddress was not called on the shim") } bridge.StartTransaction() if !shim.startTxnCalled { t.Error("StartTransaction was not called on the shim") } } Sia-1.0.4/modules/renter/contractor/dependencies.go000066400000000000000000000057001302033070500223250ustar00rootroot00000000000000package contractor import ( "path/filepath" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/types" ) // These interfaces define the HostDB's dependencies. Using the smallest // interface possible makes it easier to mock these dependencies in testing. type ( consensusSet interface { ConsensusSetSubscribe(modules.ConsensusSetSubscriber, modules.ConsensusChangeID) error Synced() bool } // in order to restrict the modules.TransactionBuilder interface, we must // provide a shim to bridge the gap between modules.Wallet and // transactionBuilder. walletShim interface { NextAddress() (types.UnlockConditions, error) StartTransaction() modules.TransactionBuilder } wallet interface { NextAddress() (types.UnlockConditions, error) StartTransaction() transactionBuilder } transactionBuilder interface { AddArbitraryData([]byte) uint64 AddFileContract(types.FileContract) uint64 AddMinerFee(types.Currency) uint64 AddParents([]types.Transaction) AddSiacoinInput(types.SiacoinInput) uint64 AddSiacoinOutput(types.SiacoinOutput) uint64 AddTransactionSignature(types.TransactionSignature) uint64 Drop() FundSiacoins(types.Currency) error Sign(bool) ([]types.Transaction, error) View() (types.Transaction, []types.Transaction) ViewAdded() (parents, coins, funds, signatures []int) } transactionPool interface { AcceptTransactionSet([]types.Transaction) error FeeEstimation() (min types.Currency, max types.Currency) } hostDB interface { Host(modules.NetAddress) (modules.HostDBEntry, bool) RandomHosts(n int, exclude []modules.NetAddress) []modules.HostDBEntry } persister interface { save(contractorPersist) error saveSync(contractorPersist) error load(*contractorPersist) error } ) // because wallet is not directly compatible with modules.Wallet (wrong // type signature for StartTransaction), we must provide a bridge type. type walletBridge struct { w walletShim } func (ws *walletBridge) NextAddress() (types.UnlockConditions, error) { return ws.w.NextAddress() } func (ws *walletBridge) StartTransaction() transactionBuilder { return ws.w.StartTransaction() } // stdPersist implements the persister interface via persist.SaveFile and // persist.LoadFile. The metadata and filename required by these functions is // internal to stdPersist. type stdPersist struct { meta persist.Metadata filename string } func (p *stdPersist) save(data contractorPersist) error { return persist.SaveFile(p.meta, data, p.filename) } func (p *stdPersist) saveSync(data contractorPersist) error { return persist.SaveFileSync(p.meta, data, p.filename) } func (p *stdPersist) load(data *contractorPersist) error { return persist.LoadFile(p.meta, data, p.filename) } func newPersist(dir string) *stdPersist { return &stdPersist{ meta: persist.Metadata{ Header: "Contractor Persistence", Version: "0.5.2", }, filename: filepath.Join(dir, "contractor.json"), } } Sia-1.0.4/modules/renter/contractor/doc.go000066400000000000000000000113431302033070500204440ustar00rootroot00000000000000/* Package contractor is responsible for forming and renewing file contracts with hosts. Its goal is to manage the low-level details of the negotiation, revision, and renewal protocols, such that the renter can operate at a higher level of abstraction. Ideally, the renter should be mostly ignorant of the Sia protocol, instead focusing on file management, redundancy, and upload/download algorithms. Contract formation does not begin until the user first calls SetAllowance. An allowance dictates how much money the contractor is allowed to spend on file contracts during a given period. When the user calls SetAllowance for the first time, the call will block until contracts have been negotiated with the specified number of hosts. Upon subsequent calls, new contracts will only be formed if the allowance is sufficiently greater than the previous allowance, where "sufficiently greater" currently means "enough money to pay for at least one additional sector on every host." This allows the user to increase the amount of available storage immediately, at the cost of some complexity. The contractor forms many contracts in parallel with different host, and tries to keep all the contracts "consistent" -- that is, they should all have the same storage capacity, and they should all end at the same height. Hosts are selected from the HostDB; there is no support for manually specifying hosts. Contracts are automatically renewed by the contractor at a safe threshold before they are set to expire. When contracts are renewed, they are renewed with the current allowance, which may differ from the allowance that was used to form the initial contracts. In general, this means that allowance modifications only take effect upon the next "contract cycle" (the exception being "sufficiently greater" modifications, as defined above). As an example, imagine that the user first sets an allowance that will cover 10 contracts of 10 sectors each for 100 blocks. The contractor will immediately form contracts with 10 hosts, paying each host enough to cover 10 sectors for 100 blocks. Then, 20 blocks later, the user increases the allowance, such that it now covers 10 contracts of 20 sectors for 200 blocks. The contractor will immediately form contracts as follows: - 10 contracts will be formed with the current hosts, each covering 10 sectors for 80 blocks. - 10 contracts will be formed with new hosts, each covering 20 sectors for 80 blocks. Note that these newly-formed contracts are timed to expire in sync with the existing contracts. This becomes the new "contract set," totaling 30 contracts, but only 20 hosts, with 20 sectors per host. When it comes time to renew these contracts, only one contract will be renewed per host, and the contracts will be renewed for the full 200-block duration. The new contract set will thus consist of 20 contracts, 20 hosts, 20 sectors, 200 blocks. On the other hand, if the allowance is decreased, no immediate action is taken. Why? Because the contracts have already been paid for. The new allowance will only take effect upon the next renewal. Modifications to file contracts are mediated through the Editor interface. An Editor maintains a network connection to a host, over which is sends modification requests, such as "delete sector 12." After each modification, the Editor revises the underlying file contract and saves it to disk. The primary challenge of the contractor is that it must be smart enough for the user to feel comfortable allowing it to spend their money. Because contract renewal is a background task, it is difficult to report errors to the user and defer to their decision. For example, what should the contractor do in the following scenarios? - The contract set is up for renewal, but the average host price has increased, and now the allowance is not sufficient to cover all of the user's uploaded data. - The user sets an allowance of 10 hosts. The contractor forms 5 contracts, but the rest fail, and the remaining hosts in the HostDB are too expensive. - After contract formation succeeds, 2 of 10 hosts become unresponsive. Later, another 4 become unresponsive. Waiting for user input is dangerous because if the contract period elapses, data is permanently lost. The contractor should treat this as the worst-case scenario, and take steps to prevent it, so long as the allowance is not exceeded. However, since the contractor has no concept of redundancy, it is not well-positioned to determine which sectors to sacrifice and which to preserve. The contractor also lacks the ability to reupload data; it can download sectors, but it does not know the decryption keys or erasure coding metadata required to reconstruct the original data. It follows that these responsibilities must be delegated to the renter. */ package contractor Sia-1.0.4/modules/renter/contractor/downloader.go000066400000000000000000000140231302033070500220330ustar00rootroot00000000000000package contractor import ( "errors" "sync" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/renter/proto" "github.com/NebulousLabs/Sia/types" ) var errInvalidDownloader = errors.New("downloader has been invalidated because its contract is being renewed") // An Downloader retrieves sectors from with a host. It requests one sector at // a time, and revises the file contract to transfer money to the host // proportional to the data retrieved. type Downloader interface { // Sector retrieves the sector with the specified Merkle root, and revises // the underlying contract to pay the host proportionally to the data // retrieve. Sector(root crypto.Hash) ([]byte, error) // Close terminates the connection to the host. Close() error } // A hostDownloader retrieves sectors by calling the download RPC on a host. // It implements the Downloader interface. hostDownloaders are safe for use by // multiple goroutines. type hostDownloader struct { clients int // safe to Close when 0 contractID types.FileContractID contractor *Contractor downloader *proto.Downloader hostSettings modules.HostExternalSettings invalid bool // true if invalidate has been called speed uint64 // Bytes per second. mu sync.Mutex } // invalidate sets the invalid flag and closes the underlying // proto.Downloader. Once invalidate returns, the hostDownloader is guaranteed // to not further revise its contract. This is used during contract renewal to // prevent a Downloader from revising a contract mid-renewal. func (hd *hostDownloader) invalidate() { hd.mu.Lock() defer hd.mu.Unlock() hd.downloader.Close() hd.invalid = true hd.contractor.mu.Lock() delete(hd.contractor.downloaders, hd.contractID) delete(hd.contractor.revising, hd.contractID) hd.contractor.mu.Unlock() } // HostSettings returns the settings of the host that the downloader connects // to. func (hd *hostDownloader) HostSettings() modules.HostExternalSettings { hd.mu.Lock() defer hd.mu.Unlock() return hd.hostSettings } // Sector retrieves the sector with the specified Merkle root, and revises // the underlying contract to pay the host proportionally to the data // retrieve. func (hd *hostDownloader) Sector(root crypto.Hash) ([]byte, error) { hd.mu.Lock() defer hd.mu.Unlock() if hd.invalid { return nil, errInvalidDownloader } oldSpending := hd.downloader.DownloadSpending start := time.Now() contract, sector, err := hd.downloader.Sector(root) duration := time.Since(start) if err != nil { return nil, err } delta := hd.downloader.DownloadSpending.Sub(oldSpending) hd.speed = uint64(duration.Seconds()) / modules.SectorSize hd.contractor.mu.Lock() hd.contractor.financialMetrics.DownloadSpending = hd.contractor.financialMetrics.DownloadSpending.Add(delta) hd.contractor.contracts[contract.ID] = contract hd.contractor.saveSync() hd.contractor.mu.Unlock() return sector, nil } // Speed returns the most recent download speed of this host, in bytes per // second. func (hd *hostDownloader) Speed() uint64 { hd.mu.Lock() defer hd.mu.Unlock() return hd.speed } // Close cleanly terminates the download loop with the host and closes the // connection. func (hd *hostDownloader) Close() error { hd.mu.Lock() defer hd.mu.Unlock() hd.clients-- // Close is a no-op if invalidate has been called, or if there are other // clients still using the hostDownloader. if hd.invalid || hd.clients > 0 { return nil } hd.contractor.mu.Lock() delete(hd.contractor.downloaders, hd.contractID) delete(hd.contractor.revising, hd.contractID) hd.contractor.mu.Unlock() return hd.downloader.Close() } // Downloader returns a Downloader object that can be used to download sectors // from a host. func (c *Contractor) Downloader(id types.FileContractID) (_ Downloader, err error) { c.mu.RLock() id = c.resolveID(id) cachedDownloader, haveDownloader := c.downloaders[id] height := c.blockHeight contract, haveContract := c.contracts[id] renewing := c.renewing[id] c.mu.RUnlock() if renewing { return nil, errors.New("currently renewing that contract") } if haveDownloader { // increment number of clients and return cachedDownloader.mu.Lock() cachedDownloader.clients++ cachedDownloader.mu.Unlock() return cachedDownloader, nil } host, haveHost := c.hdb.Host(contract.NetAddress) if !haveContract { return nil, errors.New("no record of that contract") } else if height > contract.EndHeight() { return nil, errors.New("contract has already ended") } else if !haveHost { return nil, errors.New("no record of that host") } else if host.DownloadBandwidthPrice.Cmp(maxDownloadPrice) > 0 { return nil, errTooExpensive } // acquire revising lock c.mu.Lock() alreadyRevising := c.revising[contract.ID] if alreadyRevising { c.mu.Unlock() return nil, errors.New("already revising that contract") } c.revising[contract.ID] = true c.mu.Unlock() // release lock early if function returns an error defer func() { if err != nil { c.mu.Lock() delete(c.revising, contract.ID) c.mu.Unlock() } }() // create downloader d, err := proto.NewDownloader(host, contract) if proto.IsRevisionMismatch(err) { // try again with the cached revision c.mu.RLock() cached, ok := c.cachedRevisions[contract.ID] c.mu.RUnlock() if !ok { // nothing we can do; return original error return nil, err } c.log.Printf("host %v has different revision for %v; retrying with cached revision", contract.NetAddress, contract.ID) contract.LastRevision = cached.revision d, err = proto.NewDownloader(host, contract) } if err != nil { return nil, err } // supply a SaveFn that saves the revision to the contractor's persist // (the existing revision will be overwritten when SaveFn is called) d.SaveFn = c.saveRevision(contract.ID) // cache downloader hd := &hostDownloader{ clients: 1, contractID: contract.ID, contractor: c, downloader: d, hostSettings: host.HostExternalSettings, } c.mu.Lock() c.downloaders[contract.ID] = hd c.mu.Unlock() return hd, nil } Sia-1.0.4/modules/renter/contractor/editor.go000066400000000000000000000203431302033070500211650ustar00rootroot00000000000000package contractor import ( "errors" "sync" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/renter/proto" "github.com/NebulousLabs/Sia/types" ) // the contractor will cap host's MaxCollateral setting to this value var maxUploadCollateral = types.SiacoinPrecision.Mul64(1e3).Div(modules.BlockBytesPerMonthTerabyte) // 1k SC / TB / Month var errInvalidEditor = errors.New("editor has been invalidated because its contract is being renewed") // An Editor modifies a Contract by communicating with a host. It uses the // contract revision protocol to send modification requests to the host. // Editors are the means by which the renter uploads data to hosts. type Editor interface { // Upload revises the underlying contract to store the new data. It // returns the Merkle root of the data. Upload(data []byte) (root crypto.Hash, err error) // Delete removes a sector from the underlying contract. Delete(crypto.Hash) error // Modify overwrites a sector with new data. Because the Editor does not // have access to the original sector data, the new Merkle root must be // supplied by the caller. Modify(oldRoot, newRoot crypto.Hash, offset uint64, newData []byte) error // Address returns the address of the host. Address() modules.NetAddress // ContractID returns the FileContractID of the contract. ContractID() types.FileContractID // EndHeight returns the height at which the contract ends. EndHeight() types.BlockHeight // Close terminates the connection to the host. Close() error } // A hostEditor modifies a Contract by calling the revise RPC on a host. It // implements the Editor interface. hostEditors are safe for use by // multiple goroutines. type hostEditor struct { clients int // safe to Close when 0 contract modules.RenterContract contractor *Contractor editor *proto.Editor invalid bool // true if invalidate has been called mu sync.Mutex } // invalidate sets the invalid flag and closes the underlying proto.Editor. // Once invalidate returns, the hostEditor is guaranteed to not further revise // its contract. This is used during contract renewal to prevent an Editor // from revising a contract mid-renewal. func (he *hostEditor) invalidate() { he.mu.Lock() defer he.mu.Unlock() he.editor.Close() he.invalid = true he.contractor.mu.Lock() delete(he.contractor.editors, he.contract.ID) delete(he.contractor.revising, he.contract.ID) he.contractor.mu.Unlock() } // Address returns the NetAddress of the host. func (he *hostEditor) Address() modules.NetAddress { return he.contract.NetAddress } // ContractID returns the ID of the contract being revised. func (he *hostEditor) ContractID() types.FileContractID { return he.contract.ID } // EndHeight returns the height at which the host is no longer obligated to // store the file. func (he *hostEditor) EndHeight() types.BlockHeight { return he.contract.EndHeight() } // Close cleanly terminates the revision loop with the host and closes the // connection. func (he *hostEditor) Close() error { he.mu.Lock() defer he.mu.Unlock() he.clients-- // Close is a no-op if invalidate has been called, or if there are other // clients still using the hostEditor. if he.invalid || he.clients > 0 { return nil } he.contractor.mu.Lock() delete(he.contractor.editors, he.contract.ID) delete(he.contractor.revising, he.contract.ID) he.contractor.mu.Unlock() return he.editor.Close() } // Upload negotiates a revision that adds a sector to a file contract. func (he *hostEditor) Upload(data []byte) (crypto.Hash, error) { he.mu.Lock() defer he.mu.Unlock() if he.invalid { return crypto.Hash{}, errInvalidEditor } oldUploadSpending := he.editor.UploadSpending oldStorageSpending := he.editor.StorageSpending contract, sectorRoot, err := he.editor.Upload(data) if err != nil { return crypto.Hash{}, err } uploadDelta := he.editor.UploadSpending.Sub(oldUploadSpending) storageDelta := he.editor.StorageSpending.Sub(oldStorageSpending) he.contractor.mu.Lock() he.contractor.financialMetrics.UploadSpending = he.contractor.financialMetrics.UploadSpending.Add(uploadDelta) he.contractor.financialMetrics.StorageSpending = he.contractor.financialMetrics.StorageSpending.Add(storageDelta) he.contractor.contracts[contract.ID] = contract he.contractor.saveSync() he.contractor.mu.Unlock() he.contract = contract return sectorRoot, nil } // Delete negotiates a revision that removes a sector from a file contract. func (he *hostEditor) Delete(root crypto.Hash) error { he.mu.Lock() defer he.mu.Unlock() if he.invalid { return errInvalidEditor } contract, err := he.editor.Delete(root) if err != nil { return err } he.contractor.mu.Lock() he.contractor.contracts[contract.ID] = contract he.contractor.saveSync() he.contractor.mu.Unlock() he.contract = contract return nil } // Modify negotiates a revision that edits a sector in a file contract. func (he *hostEditor) Modify(oldRoot, newRoot crypto.Hash, offset uint64, newData []byte) error { he.mu.Lock() defer he.mu.Unlock() if he.invalid { return errInvalidEditor } oldUploadSpending := he.editor.UploadSpending contract, err := he.editor.Modify(oldRoot, newRoot, offset, newData) if err != nil { return err } uploadDelta := he.editor.UploadSpending.Sub(oldUploadSpending) he.contractor.mu.Lock() he.contractor.financialMetrics.UploadSpending = he.contractor.financialMetrics.UploadSpending.Add(uploadDelta) he.contractor.contracts[contract.ID] = contract he.contractor.saveSync() he.contractor.mu.Unlock() he.contract = contract return nil } // Editor returns a Editor object that can be used to upload, modify, and // delete sectors on a host. func (c *Contractor) Editor(id types.FileContractID) (_ Editor, err error) { c.mu.RLock() id = c.resolveID(id) cachedEditor, haveEditor := c.editors[id] height := c.blockHeight contract, haveContract := c.contracts[id] renewing := c.renewing[id] c.mu.RUnlock() if renewing { return nil, errors.New("currently renewing that contract") } if haveEditor { // increment number of clients and return cachedEditor.mu.Lock() cachedEditor.clients++ cachedEditor.mu.Unlock() return cachedEditor, nil } host, haveHost := c.hdb.Host(contract.NetAddress) if !haveContract { return nil, errors.New("no record of that contract") } else if height > contract.EndHeight() { return nil, errors.New("contract has already ended") } else if !haveHost { return nil, errors.New("no record of that host") } else if host.StoragePrice.Cmp(maxStoragePrice) > 0 { return nil, errTooExpensive } else if build.VersionCmp(host.Version, "0.6.0") > 0 { // COMPATv0.6.0: don't cap host.Collateral on old hosts if host.Collateral.Cmp(maxUploadCollateral) > 0 { host.Collateral = maxUploadCollateral } } // acquire revising lock c.mu.Lock() alreadyRevising := c.revising[contract.ID] if alreadyRevising { c.mu.Unlock() return nil, errors.New("already revising that contract") } c.revising[contract.ID] = true c.mu.Unlock() // release lock early if function returns an error defer func() { if err != nil { c.mu.Lock() delete(c.revising, contract.ID) c.mu.Unlock() } }() // create editor e, err := proto.NewEditor(host, contract, height) if proto.IsRevisionMismatch(err) { // try again with the cached revision c.mu.RLock() cached, ok := c.cachedRevisions[contract.ID] c.mu.RUnlock() if !ok { // nothing we can do; return original error c.log.Printf("wanted to recover contract %v with host %v, but no revision was cached", contract.ID, contract.NetAddress) return nil, err } c.log.Printf("host %v has different revision for %v; retrying with cached revision", contract.NetAddress, contract.ID) contract.LastRevision = cached.revision contract.MerkleRoots = cached.merkleRoots e, err = proto.NewEditor(host, contract, height) } if err != nil { return nil, err } // supply a SaveFn that saves the revision to the contractor's persist // (the existing revision will be overwritten when SaveFn is called) e.SaveFn = c.saveRevision(contract.ID) // cache editor he := &hostEditor{ clients: 1, contract: contract, contractor: c, editor: e, } c.mu.Lock() c.editors[contract.ID] = he c.mu.Unlock() return he, nil } Sia-1.0.4/modules/renter/contractor/formcontract.go000066400000000000000000000137161302033070500224060ustar00rootroot00000000000000package contractor import ( "errors" "fmt" "strings" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/renter/proto" "github.com/NebulousLabs/Sia/types" ) var ( // the contractor will not form contracts above this price maxStoragePrice = types.SiacoinPrecision.Mul64(500e3).Div(modules.BlockBytesPerMonthTerabyte) // 500k SC / TB / Month // the contractor will not download data above this price (3x the maximum monthly storage price) maxDownloadPrice = maxStoragePrice.Mul64(3 * 4320) // the contractor will cap host's MaxCollateral setting to this value maxCollateral = types.SiacoinPrecision.Mul64(1e3) // 1k SC // ErrInsufficientAllowance indicates that the renter's allowance is less // than the amount necessary to store at least one sector ErrInsufficientAllowance = errors.New("allowance is not large enough to cover fees of contract creation") errTooExpensive = errors.New("host price was too high") ) // maxSectors is the estimated maximum number of sectors that the allowance // can support. func maxSectors(a modules.Allowance, hdb hostDB, tp transactionPool) (uint64, error) { if a.Hosts <= 0 || a.Period <= 0 { return 0, errors.New("invalid allowance") } // Sample at least 10 hosts. nRandomHosts := int(a.Hosts) if nRandomHosts < minHostsForEstimations { nRandomHosts = minHostsForEstimations } hosts := hdb.RandomHosts(nRandomHosts, nil) if len(hosts) < int(a.Hosts) { return 0, fmt.Errorf("not enough hosts in hostdb for sector calculation, got %v but needed %v", len(hosts), int(a.Hosts)) } // Calculate cost of creating contracts with each host, and the cost of // storing sectors on each host. var sectorSum types.Currency var contractCostSum types.Currency for _, h := range hosts { sectorSum = sectorSum.Add(h.StoragePrice) contractCostSum = contractCostSum.Add(h.ContractPrice) } averageSectorPrice := sectorSum.Div64(uint64(len(hosts))) averageContractPrice := contractCostSum.Div64(uint64(len(hosts))) costPerSector := averageSectorPrice.Mul64(a.Hosts).Mul64(modules.SectorSize).Mul64(uint64(a.Period)) costForContracts := averageContractPrice.Mul64(a.Hosts) // Subtract fees for creating the file contracts from the allowance. _, feeEstimation := tp.FeeEstimation() costForTxnFees := types.NewCurrency64(estimatedFileContractTransactionSize).Mul(feeEstimation).Mul64(a.Hosts) // Check for potential divide by zero if a.Funds.Cmp(costForTxnFees.Add(costForContracts)) <= 0 { return 0, ErrInsufficientAllowance } sectorFunds := a.Funds.Sub(costForTxnFees).Sub(costForContracts) // Divide total funds by cost per sector. numSectors, err := sectorFunds.Div(costPerSector).Uint64() if err != nil { return 0, errors.New("error when totaling number of sectors that can be bought with an allowance: " + err.Error()) } return numSectors, nil } // managedNewContract negotiates an initial file contract with the specified // host, saves it, and returns it. func (c *Contractor) managedNewContract(host modules.HostDBEntry, numSectors uint64, endHeight types.BlockHeight) (modules.RenterContract, error) { // reject hosts that are too expensive if host.StoragePrice.Cmp(maxStoragePrice) > 0 { return modules.RenterContract{}, errTooExpensive } // cap host.MaxCollateral if host.MaxCollateral.Cmp(maxCollateral) > 0 { host.MaxCollateral = maxCollateral } // get an address to use for negotiation uc, err := c.wallet.NextAddress() if err != nil { return modules.RenterContract{}, err } // create contract params c.mu.RLock() params := proto.ContractParams{ Host: host, Filesize: numSectors * modules.SectorSize, StartHeight: c.blockHeight, EndHeight: endHeight, RefundAddress: uc.UnlockHash(), } c.mu.RUnlock() // create transaction builder txnBuilder := c.wallet.StartTransaction() contract, err := proto.FormContract(params, txnBuilder, c.tpool) if err != nil { txnBuilder.Drop() return modules.RenterContract{}, err } contractValue := contract.RenterFunds() c.log.Printf("Formed contract with %v for %v SC", host.NetAddress, contractValue.Div(types.SiacoinPrecision)) return contract, nil } // managedFormContracts forms contracts with n hosts using the allowance // parameters. func (c *Contractor) managedFormContracts(n int, numSectors uint64, endHeight types.BlockHeight) ([]modules.RenterContract, error) { if n <= 0 { return nil, nil } // Sample at least 10 hosts. nRandomHosts := 2 * n if nRandomHosts < 10 { nRandomHosts = 10 } // Don't select from hosts we've already formed contracts with c.mu.RLock() var exclude []modules.NetAddress for _, contract := range c.contracts { exclude = append(exclude, contract.NetAddress) } c.mu.RUnlock() hosts := c.hdb.RandomHosts(nRandomHosts, exclude) if len(hosts) < n { return nil, fmt.Errorf("not enough hosts in hostdb for contract formation, got %v but needed %v", len(hosts), n) } var contracts []modules.RenterContract var errs []string for _, h := range hosts { contract, err := c.managedNewContract(h, numSectors, endHeight) if err != nil { errs = append(errs, fmt.Sprintf("\t%v: %v", h.NetAddress, err)) continue } contracts = append(contracts, contract) if len(contracts) >= n { break } if build.Release != "testing" { // sleep for 1 minute to alleviate potential block propagation issues time.Sleep(60 * time.Second) } } // If we couldn't form any contracts, return an error. Otherwise, just log // the failures. // TODO: is there a better way to handle failure here? Should we prefer an // all-or-nothing approach? We can't pick new hosts to negotiate with // because they'll probably be more expensive than we can afford. if len(contracts) == 0 { return nil, errors.New("could not form any contracts:\n" + strings.Join(errs, "\n")) } else if len(contracts) < n { c.log.Printf("WARN: failed to form desired number of contracts (wanted %v, got %v):\n%v", n, len(contracts), strings.Join(errs, "\n")) } return contracts, nil } Sia-1.0.4/modules/renter/contractor/host_integration_test.go000066400000000000000000000531151302033070500243210ustar00rootroot00000000000000package contractor import ( "bytes" "errors" "os" "path/filepath" "testing" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/consensus" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/modules/host" "github.com/NebulousLabs/Sia/modules/miner" "github.com/NebulousLabs/Sia/modules/renter/hostdb" "github.com/NebulousLabs/Sia/modules/renter/proto" "github.com/NebulousLabs/Sia/modules/transactionpool" modWallet "github.com/NebulousLabs/Sia/modules/wallet" "github.com/NebulousLabs/Sia/types" ) // newTestingWallet is a helper function that creates a ready-to-use wallet // and mines some coins into it. func newTestingWallet(testdir string, cs modules.ConsensusSet, tp modules.TransactionPool) (modules.Wallet, error) { w, err := modWallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } key, err := crypto.GenerateTwofishKey() if err != nil { return nil, err } if !w.Encrypted() { _, err = w.Encrypt(key) if err != nil { return nil, err } } err = w.Unlock(key) if err != nil { return nil, err } // give it some money m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.MinerDir)) if err != nil { return nil, err } for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ { _, err := m.AddBlock() if err != nil { return nil, err } } return w, nil } // newTestingHost is a helper function that creates a ready-to-use host. func newTestingHost(testdir string, cs modules.ConsensusSet, tp modules.TransactionPool) (modules.Host, error) { w, err := newTestingWallet(testdir, cs, tp) if err != nil { return nil, err } h, err := host.New(cs, tp, w, "localhost:0", filepath.Join(testdir, modules.HostDir)) if err != nil { return nil, err } // configure host to accept contracts settings := h.InternalSettings() settings.AcceptingContracts = true err = h.SetInternalSettings(settings) if err != nil { return nil, err } // add storage to host storageFolder := filepath.Join(testdir, "storage") err = os.MkdirAll(storageFolder, 0700) if err != nil { return nil, err } err = h.AddStorageFolder(storageFolder, 1e6) if err != nil { return nil, err } return h, nil } // newTestingContractor is a helper function that creates a ready-to-use // contractor. func newTestingContractor(testdir string, cs modules.ConsensusSet, tp modules.TransactionPool) (*Contractor, error) { w, err := newTestingWallet(testdir, cs, tp) if err != nil { return nil, err } hdb, err := hostdb.New(cs, filepath.Join(testdir, "hostdb")) if err != nil { return nil, err } return New(cs, w, tp, hdb, filepath.Join(testdir, "contractor")) } // newTestingTrio creates a Host, Contractor, and TestMiner that can be used // for testing host/renter interactions. func newTestingTrio(name string) (modules.Host, *Contractor, modules.TestMiner, error) { testdir := build.TempDir("contractor", name) // create miner g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, nil, nil, err } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, nil, nil, err } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { return nil, nil, nil, err } w, err := modWallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, nil, nil, err } key, err := crypto.GenerateTwofishKey() if err != nil { return nil, nil, nil, err } if !w.Encrypted() { _, err = w.Encrypt(key) if err != nil { return nil, nil, nil, err } } err = w.Unlock(key) if err != nil { return nil, nil, nil, err } m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.MinerDir)) if err != nil { return nil, nil, nil, err } // create host and contractor, using same consensus set and gateway h, err := newTestingHost(filepath.Join(testdir, "Host"), cs, tp) if err != nil { return nil, nil, nil, err } c, err := newTestingContractor(filepath.Join(testdir, "Contractor"), cs, tp) if err != nil { return nil, nil, nil, err } // announce the host err = h.Announce() if err != nil { return nil, nil, nil, err } // mine a block, processing the announcement m.AddBlock() // wait for hostdb to scan host for i := 0; i < 100 && len(c.hdb.RandomHosts(1, nil)) == 0; i++ { time.Sleep(time.Millisecond * 50) } if len(c.hdb.RandomHosts(1, nil)) == 0 { return nil, nil, nil, errors.New("host did not make it into the contractor hostdb in time") } return h, c, m, nil } // TestIntegrationFormContract tests that the contractor can form contracts // with the host module. func TestIntegrationFormContract(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() h, c, _, err := newTestingTrio("TestIntegrationFormContract") if err != nil { t.Fatal(err) } defer h.Close() // get the host's entry from the db hostEntry, ok := c.hdb.Host(h.ExternalSettings().NetAddress) if !ok { t.Fatal("no entry for host in db") } // form a contract with the host contract, err := c.managedNewContract(hostEntry, 10, c.blockHeight+100) if err != nil { t.Fatal(err) } if contract.NetAddress != h.ExternalSettings().NetAddress { t.Fatal("bad contract") } } // TestIntegrationReviseContract tests that the contractor can revise a // contract previously formed with a host. func TestIntegrationReviseContract(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // create testing trio h, c, _, err := newTestingTrio("TestIntegrationReviseContract") if err != nil { t.Fatal(err) } defer h.Close() // get the host's entry from the db hostEntry, ok := c.hdb.Host(h.ExternalSettings().NetAddress) if !ok { t.Fatal("no entry for host in db") } // form a contract with the host contract, err := c.managedNewContract(hostEntry, 10, c.blockHeight+100) if err != nil { t.Fatal(err) } c.mu.Lock() c.contracts[contract.ID] = contract c.mu.Unlock() // revise the contract editor, err := c.Editor(contract.ID) if err != nil { t.Fatal(err) } data, err := crypto.RandBytes(int(modules.SectorSize)) if err != nil { t.Fatal(err) } _, err = editor.Upload(data) if err != nil { t.Fatal(err) } err = editor.Close() if err != nil { t.Fatal(err) } } // TestIntegrationUploadDownload tests that the contractor can upload data to // a host and download it intact. func TestIntegrationUploadDownload(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // create testing trio h, c, _, err := newTestingTrio("TestIntegrationUploadDownload") if err != nil { t.Fatal(err) } defer h.Close() // get the host's entry from the db hostEntry, ok := c.hdb.Host(h.ExternalSettings().NetAddress) if !ok { t.Fatal("no entry for host in db") } // form a contract with the host contract, err := c.managedNewContract(hostEntry, 10, c.blockHeight+100) if err != nil { t.Fatal(err) } c.mu.Lock() c.contracts[contract.ID] = contract c.mu.Unlock() // revise the contract editor, err := c.Editor(contract.ID) if err != nil { t.Fatal(err) } data, err := crypto.RandBytes(int(modules.SectorSize)) if err != nil { t.Fatal(err) } root, err := editor.Upload(data) if err != nil { t.Fatal(err) } err = editor.Close() if err != nil { t.Fatal(err) } // download the data downloader, err := c.Downloader(contract.ID) if err != nil { t.Fatal(err) } retrieved, err := downloader.Sector(root) if err != nil { t.Fatal(err) } if !bytes.Equal(data, retrieved) { t.Fatal("downloaded data does not match original") } err = downloader.Close() if err != nil { t.Fatal(err) } } // TestIntegrationDelete tests that the contractor can delete a sector from a // contract previously formed with a host. func TestIntegrationDelete(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // create testing trio h, c, _, err := newTestingTrio("TestIntegrationDelete") if err != nil { t.Fatal(err) } defer h.Close() // get the host's entry from the db hostEntry, ok := c.hdb.Host(h.ExternalSettings().NetAddress) if !ok { t.Fatal("no entry for host in db") } // form a contract with the host contract, err := c.managedNewContract(hostEntry, 10, c.blockHeight+100) if err != nil { t.Fatal(err) } c.mu.Lock() c.contracts[contract.ID] = contract c.mu.Unlock() // revise the contract editor, err := c.Editor(contract.ID) if err != nil { t.Fatal(err) } data, err := crypto.RandBytes(int(modules.SectorSize)) if err != nil { t.Fatal(err) } _, err = editor.Upload(data) if err != nil { t.Fatal(err) } err = editor.Close() if err != nil { t.Fatal(err) } c.mu.Lock() contract = c.contracts[contract.ID] c.mu.Unlock() // delete the sector editor, err = c.Editor(contract.ID) if err != nil { t.Fatal(err) } err = editor.Delete(contract.MerkleRoots[0]) if err != nil { t.Fatal(err) } err = editor.Close() if err != nil { t.Fatal(err) } } // TestIntegrationInsertDelete tests that the contractor can insert and delete // a sector during the same revision. func TestIntegrationInsertDelete(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // create testing trio h, c, _, err := newTestingTrio("TestIntegrationInsertDelete") if err != nil { t.Fatal(err) } defer h.Close() // get the host's entry from the db hostEntry, ok := c.hdb.Host(h.ExternalSettings().NetAddress) if !ok { t.Fatal("no entry for host in db") } // form a contract with the host contract, err := c.managedNewContract(hostEntry, 10, c.blockHeight+100) if err != nil { t.Fatal(err) } c.mu.Lock() c.contracts[contract.ID] = contract c.mu.Unlock() // revise the contract editor, err := c.Editor(contract.ID) if err != nil { t.Fatal(err) } data, err := crypto.RandBytes(int(modules.SectorSize)) if err != nil { t.Fatal(err) } // insert the sector _, err = editor.Upload(data) if err != nil { t.Fatal(err) } // delete the sector err = editor.Delete(crypto.MerkleRoot(data)) if err != nil { t.Fatal(err) } err = editor.Close() if err != nil { t.Fatal(err) } // contract should have no sectors contract = c.contracts[contract.ID] if len(contract.MerkleRoots) != 0 { t.Fatal("contract should have no sectors:", contract.MerkleRoots) } } // TestIntegrationModify tests that the contractor can modify a previously- // uploaded sector. func TestIntegrationModify(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // create testing trio h, c, _, err := newTestingTrio("TestIntegrationModify") if err != nil { t.Fatal(err) } defer h.Close() // get the host's entry from the db hostEntry, ok := c.hdb.Host(h.ExternalSettings().NetAddress) if !ok { t.Fatal("no entry for host in db") } // form a contract with the host contract, err := c.managedNewContract(hostEntry, 10, c.blockHeight+100) if err != nil { t.Fatal(err) } c.mu.Lock() c.contracts[contract.ID] = contract c.mu.Unlock() // revise the contract editor, err := c.Editor(contract.ID) if err != nil { t.Fatal(err) } data, err := crypto.RandBytes(int(modules.SectorSize)) if err != nil { t.Fatal(err) } // insert the sector _, err = editor.Upload(data) if err != nil { t.Fatal(err) } err = editor.Close() if err != nil { t.Fatal(err) } // modify the sector oldRoot := crypto.MerkleRoot(data) offset, newData := uint64(10), []byte{1, 2, 3, 4, 5} copy(data[offset:], newData) newRoot := crypto.MerkleRoot(data) editor, err = c.Editor(contract.ID) if err != nil { t.Fatal(err) } err = editor.Modify(oldRoot, newRoot, offset, newData) if err != nil { t.Fatal(err) } err = editor.Close() if err != nil { t.Fatal(err) } } // TestIntegrationRenew tests that the contractor can renew a previously- // formed file contract. func TestIntegrationRenew(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // create testing trio h, c, _, err := newTestingTrio("TestIntegrationRenew") if err != nil { t.Fatal(err) } defer h.Close() // get the host's entry from the db hostEntry, ok := c.hdb.Host(h.ExternalSettings().NetAddress) if !ok { t.Fatal("no entry for host in db") } // form a contract with the host contract, err := c.managedNewContract(hostEntry, 10, c.blockHeight+100) if err != nil { t.Fatal(err) } c.mu.Lock() c.contracts[contract.ID] = contract c.mu.Unlock() // revise the contract editor, err := c.Editor(contract.ID) if err != nil { t.Fatal(err) } data, err := crypto.RandBytes(int(modules.SectorSize)) if err != nil { t.Fatal(err) } // insert the sector root, err := editor.Upload(data) if err != nil { t.Fatal(err) } err = editor.Close() if err != nil { t.Fatal(err) } // renew the contract oldContract := c.contracts[contract.ID] contract, err = c.managedRenew(oldContract, modules.SectorSize*10, c.blockHeight+200) if err != nil { t.Fatal(err) } c.mu.Lock() c.contracts[contract.ID] = contract c.mu.Unlock() // check renewed contract if contract.FileContract.FileMerkleRoot != root { t.Fatal(contract.FileContract.FileMerkleRoot) } else if contract.FileContract.FileSize != modules.SectorSize { t.Fatal(contract.FileContract.FileSize) } else if contract.FileContract.RevisionNumber != 0 { t.Fatal(contract.FileContract.RevisionNumber) } else if contract.FileContract.WindowStart != c.blockHeight+200 { t.Fatal(contract.FileContract.WindowStart) } // check that Merkle roots are intact if len(contract.MerkleRoots) != len(oldContract.MerkleRoots) { t.Fatal(len(contract.MerkleRoots), len(oldContract.MerkleRoots)) } // download the renewed contract downloader, err := c.Downloader(contract.ID) if err != nil { t.Fatal(err) } retrieved, err := downloader.Sector(root) if err != nil { t.Fatal(err) } if !bytes.Equal(data, retrieved) { t.Fatal("downloaded data does not match original") } err = downloader.Close() if err != nil { t.Fatal(err) } // renew to a lower height oldContract = c.contracts[contract.ID] contract, err = c.managedRenew(oldContract, modules.SectorSize*10, c.blockHeight+100) if err != nil { t.Fatal(err) } if contract.FileContract.WindowStart != c.blockHeight+100 { t.Fatal(contract.FileContract.WindowStart) } // check that Merkle roots are intact if len(contract.MerkleRoots) != len(oldContract.MerkleRoots) { t.Fatal(len(contract.MerkleRoots), len(oldContract.MerkleRoots)) } c.mu.Lock() c.contracts[contract.ID] = contract c.mu.Unlock() // revise the contract editor, err = c.Editor(contract.ID) if err != nil { t.Fatal(err) } data, err = crypto.RandBytes(int(modules.SectorSize)) if err != nil { t.Fatal(err) } // insert the sector _, err = editor.Upload(data) if err != nil { t.Fatal(err) } err = editor.Close() if err != nil { t.Fatal(err) } } // TestIntegrationResync tests that the contractor can resync with a host // after being interrupted during contract revision. func TestIntegrationResync(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // create testing trio h, c, _, err := newTestingTrio("TestIntegrationResync") if err != nil { t.Fatal(err) } defer h.Close() // get the host's entry from the db hostEntry, ok := c.hdb.Host(h.ExternalSettings().NetAddress) if !ok { t.Fatal("no entry for host in db") } // form a contract with the host contract, err := c.managedNewContract(hostEntry, 10, c.blockHeight+100) if err != nil { t.Fatal(err) } c.mu.Lock() c.contracts[contract.ID] = contract c.mu.Unlock() // revise the contract editor, err := c.Editor(contract.ID) if err != nil { t.Fatal(err) } data, err := crypto.RandBytes(int(modules.SectorSize)) if err != nil { t.Fatal(err) } root, err := editor.Upload(data) if err != nil { t.Fatal(err) } err = editor.Close() if err != nil { t.Fatal(err) } // download the data downloader, err := c.Downloader(contract.ID) if err != nil { t.Fatal(err) } retrieved, err := downloader.Sector(root) if err != nil { t.Fatal(err) } if !bytes.Equal(data, retrieved) { t.Fatal("downloaded data does not match original") } err = downloader.Close() if err != nil { t.Fatal(err) } contract = c.contracts[contract.ID] // corrupt contract and delete its cachedRevision badContract := contract badContract.LastRevision.NewRevisionNumber-- badContract.LastRevisionTxn.TransactionSignatures = nil // delete signatures c.mu.Lock() delete(c.cachedRevisions, contract.ID) c.mu.Unlock() c.mu.Lock() c.contracts[badContract.ID] = badContract c.mu.Unlock() // Editor should fail with the bad contract _, err = c.Editor(badContract.ID) if !proto.IsRevisionMismatch(err) { t.Fatal("expected revision mismatch, got", err) } // add cachedRevision cachedRev := cachedRevision{contract.LastRevision, contract.MerkleRoots} c.mu.Lock() c.cachedRevisions[contract.ID] = cachedRev c.mu.Unlock() // Editor and Downloader should now succeed after loading the cachedRevision editor, err = c.Editor(badContract.ID) if err != nil { t.Fatal(err) } editor.Close() downloader, err = c.Downloader(badContract.ID) if err != nil { t.Fatal(err) } downloader.Close() // corrupt contract and delete its cachedRevision badContract = contract badContract.LastRevision.NewRevisionNumber-- badContract.MerkleRoots = nil // delete Merkle roots c.mu.Lock() c.contracts[badContract.ID] = badContract delete(c.cachedRevisions, contract.ID) c.mu.Unlock() // Editor should fail with the bad contract _, err = c.Editor(badContract.ID) if !proto.IsRevisionMismatch(err) { t.Fatal("expected revision mismatch, got", err) } // add cachedRevision c.mu.Lock() c.cachedRevisions[contract.ID] = cachedRev c.mu.Unlock() // should be able to upload after loading the cachedRevision editor, err = c.Editor(badContract.ID) if err != nil { t.Fatal(err) } _, err = editor.Upload(data) if err != nil { t.Fatal(err) } editor.Close() } // TestIntegrationDownloaderCaching tests that downloaders are properly cached // by the contractor. When two downloaders are requested for the same // contract, only one underlying downloader should be created. func TestIntegrationDownloaderCaching(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // create testing trio h, c, _, err := newTestingTrio("TestIntegrationDownloaderCaching") if err != nil { t.Fatal(err) } defer h.Close() // get the host's entry from the db hostEntry, ok := c.hdb.Host(h.ExternalSettings().NetAddress) if !ok { t.Fatal("no entry for host in db") } // form a contract with the host contract, err := c.managedNewContract(hostEntry, 10, c.blockHeight+100) if err != nil { t.Fatal(err) } c.mu.Lock() c.contracts[contract.ID] = contract c.mu.Unlock() // create a downloader d1, err := c.Downloader(contract.ID) if err != nil { t.Fatal(err) } // create another downloader d2, err := c.Downloader(contract.ID) if err != nil { t.Fatal(err) } // downloaders should match if d1 != d2 { t.Fatal("downloader was not cached") } // close one of the downloaders; it should not fully close, since d1 is // still using it d2.Close() c.mu.RLock() _, ok = c.downloaders[contract.ID] c.mu.RUnlock() if !ok { t.Fatal("expected downloader to still be present") } // create another downloader d3, err := c.Downloader(contract.ID) if err != nil { t.Fatal(err) } // downloaders should match if d3 != d1 { t.Fatal("closing one client should not fully close the downloader") } // close both downloaders d1.Close() d2.Close() c.mu.RLock() _, ok = c.downloaders[contract.ID] c.mu.RUnlock() if ok { t.Fatal("did not expect downloader to still be present") } // create another downloader d4, err := c.Downloader(contract.ID) if err != nil { t.Fatal(err) } // downloaders should match if d4 == d1 { t.Fatal("downloader should not have been cached after all clients were closed") } d4.Close() } // TestIntegrationEditorCaching tests that editors are properly cached // by the contractor. When two editors are requested for the same // contract, only one underlying editor should be created. func TestIntegrationEditorCaching(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // create testing trio h, c, _, err := newTestingTrio("TestIntegrationEditorCaching") if err != nil { t.Fatal(err) } defer h.Close() // get the host's entry from the db hostEntry, ok := c.hdb.Host(h.ExternalSettings().NetAddress) if !ok { t.Fatal("no entry for host in db") } // form a contract with the host contract, err := c.managedNewContract(hostEntry, 10, c.blockHeight+100) if err != nil { t.Fatal(err) } c.mu.Lock() c.contracts[contract.ID] = contract c.mu.Unlock() // create an editor d1, err := c.Editor(contract.ID) if err != nil { t.Fatal(err) } // create another editor d2, err := c.Editor(contract.ID) if err != nil { t.Fatal(err) } // editors should match if d1 != d2 { t.Fatal("editor was not cached") } // close one of the editors; it should not fully close, since d1 is // still using it d2.Close() c.mu.RLock() _, ok = c.editors[contract.ID] c.mu.RUnlock() if !ok { t.Fatal("expected editor to still be present") } // create another editor d3, err := c.Editor(contract.ID) if err != nil { t.Fatal(err) } // editors should match if d3 != d1 { t.Fatal("closing one client should not fully close the editor") } // close both editors d1.Close() d2.Close() c.mu.RLock() _, ok = c.editors[contract.ID] c.mu.RUnlock() if ok { t.Fatal("did not expect editor to still be present") } // create another editor d4, err := c.Editor(contract.ID) if err != nil { t.Fatal(err) } // editors should match if d4 == d1 { t.Fatal("editor should not have been cached after all clients were closed") } d4.Close() } Sia-1.0.4/modules/renter/contractor/negotiate_test.go000066400000000000000000000153421302033070500227200ustar00rootroot00000000000000package contractor import ( "path/filepath" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/consensus" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/modules/miner" "github.com/NebulousLabs/Sia/modules/renter/hostdb" "github.com/NebulousLabs/Sia/modules/transactionpool" modWallet "github.com/NebulousLabs/Sia/modules/wallet" // name conflicts with type "github.com/NebulousLabs/Sia/types" ) // contractorTester contains all of the modules that are used while testing the contractor. type contractorTester struct { cs modules.ConsensusSet gateway modules.Gateway miner modules.TestMiner tpool modules.TransactionPool wallet modules.Wallet walletKey crypto.TwofishKey hdb hostDB contractor *Contractor } // Close shuts down the contractor tester. func (rt *contractorTester) Close() error { rt.wallet.Lock() rt.cs.Close() rt.gateway.Close() return nil } // newContractorTester creates a ready-to-use contractor tester with money in the // wallet. func newContractorTester(name string) (*contractorTester, error) { // Create the modules. testdir := build.TempDir("contractor", name) g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, err } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { return nil, err } w, err := modWallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } key, err := crypto.GenerateTwofishKey() if err != nil { return nil, err } _, err = w.Encrypt(key) if err != nil { return nil, err } err = w.Unlock(key) if err != nil { return nil, err } hdb, err := hostdb.New(cs, filepath.Join(testdir, modules.RenterDir)) if err != nil { return nil, err } m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.MinerDir)) if err != nil { return nil, err } c, err := New(cs, w, tp, hdb, filepath.Join(testdir, modules.RenterDir)) if err != nil { return nil, err } // Assemble all pieces into a contractor tester. ct := &contractorTester{ cs: cs, gateway: g, miner: m, tpool: tp, wallet: w, hdb: hdb, contractor: c, } // Mine blocks until there is money in the wallet. for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ { _, err := ct.miner.AddBlock() if err != nil { return nil, err } } return ct, nil } func TestNegotiateContract(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ct, err := newContractorTester("TestNegotiateContract") if err != nil { t.Fatal(err) } payout := types.NewCurrency64(1e16) fc := types.FileContract{ FileSize: 0, FileMerkleRoot: crypto.Hash{}, // no proof possible without data WindowStart: 100, WindowEnd: 1000, Payout: payout, ValidProofOutputs: []types.SiacoinOutput{ {Value: types.PostTax(ct.contractor.blockHeight, payout), UnlockHash: types.UnlockHash{}}, {Value: types.ZeroCurrency, UnlockHash: types.UnlockHash{}}, }, MissedProofOutputs: []types.SiacoinOutput{ // same as above {Value: types.PostTax(ct.contractor.blockHeight, payout), UnlockHash: types.UnlockHash{}}, // goes to the void, not the hostdb {Value: types.ZeroCurrency, UnlockHash: types.UnlockHash{}}, }, UnlockHash: types.UnlockHash{}, RevisionNumber: 0, } txnBuilder := ct.wallet.StartTransaction() err = txnBuilder.FundSiacoins(fc.Payout) if err != nil { t.Fatal(err) } txnBuilder.AddFileContract(fc) signedTxnSet, err := txnBuilder.Sign(true) if err != nil { t.Fatal(err) } err = ct.tpool.AcceptTransactionSet(signedTxnSet) if err != nil { t.Fatal(err) } } func TestReviseContract(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() ct, err := newContractorTester("TestReviseContract") if err != nil { t.Fatal(err) } // get an address ourAddr, err := ct.wallet.NextAddress() if err != nil { t.Fatal(err) } // generate keys sk, pk, err := crypto.GenerateKeyPair() if err != nil { t.Fatal(err) } renterPubKey := types.SiaPublicKey{ Algorithm: types.SignatureEd25519, Key: pk[:], } uc := types.UnlockConditions{ PublicKeys: []types.SiaPublicKey{renterPubKey, renterPubKey}, SignaturesRequired: 1, } // create file contract payout := types.NewCurrency64(1e16) fc := types.FileContract{ FileSize: 0, FileMerkleRoot: crypto.Hash{}, // no proof possible without data WindowStart: 100, WindowEnd: 1000, Payout: payout, UnlockHash: uc.UnlockHash(), RevisionNumber: 0, } // outputs need account for tax fc.ValidProofOutputs = []types.SiacoinOutput{ {Value: types.PostTax(ct.contractor.blockHeight, payout), UnlockHash: ourAddr.UnlockHash()}, {Value: types.ZeroCurrency, UnlockHash: types.UnlockHash{}}, // no collateral } fc.MissedProofOutputs = []types.SiacoinOutput{ // same as above fc.ValidProofOutputs[0], // goes to the void, not the hostdb {Value: types.ZeroCurrency, UnlockHash: types.UnlockHash{}}, } txnBuilder := ct.wallet.StartTransaction() err = txnBuilder.FundSiacoins(fc.Payout) if err != nil { t.Fatal(err) } txnBuilder.AddFileContract(fc) signedTxnSet, err := txnBuilder.Sign(true) if err != nil { t.Fatal(err) } // submit contract err = ct.tpool.AcceptTransactionSet(signedTxnSet) if err != nil { t.Fatal(err) } // create revision fcid := signedTxnSet[len(signedTxnSet)-1].FileContractID(0) rev := types.FileContractRevision{ ParentID: fcid, UnlockConditions: uc, NewFileSize: 10, NewWindowStart: 100, NewWindowEnd: 1000, NewRevisionNumber: 1, NewValidProofOutputs: fc.ValidProofOutputs, NewMissedProofOutputs: fc.MissedProofOutputs, } // create transaction containing the revision signedTxn := types.Transaction{ FileContractRevisions: []types.FileContractRevision{rev}, TransactionSignatures: []types.TransactionSignature{{ ParentID: crypto.Hash(fcid), CoveredFields: types.CoveredFields{FileContractRevisions: []uint64{0}}, PublicKeyIndex: 0, // hostdb key is always first -- see negotiateContract }}, } // sign the transaction encodedSig, err := crypto.SignHash(signedTxn.SigHash(0), sk) if err != nil { t.Fatal(err) } signedTxn.TransactionSignatures[0].Signature = encodedSig[:] err = signedTxn.StandaloneValid(ct.contractor.blockHeight) if err != nil { t.Fatal(err) } // submit revision err = ct.tpool.AcceptTransactionSet([]types.Transaction{signedTxn}) if err != nil { t.Fatal(err) } } Sia-1.0.4/modules/renter/contractor/persist.go000066400000000000000000000052321302033070500213700ustar00rootroot00000000000000package contractor import ( "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // contractorPersist defines what Contractor data persists across sessions. type contractorPersist struct { Allowance modules.Allowance BlockHeight types.BlockHeight CachedRevisions []cachedRevision Contracts []modules.RenterContract FinancialMetrics modules.RenterFinancialMetrics LastChange modules.ConsensusChangeID RenewedIDs map[string]string } // persistData returns the data in the Contractor that will be saved to disk. func (c *Contractor) persistData() contractorPersist { data := contractorPersist{ Allowance: c.allowance, BlockHeight: c.blockHeight, FinancialMetrics: c.financialMetrics, LastChange: c.lastChange, RenewedIDs: make(map[string]string), } for _, rev := range c.cachedRevisions { data.CachedRevisions = append(data.CachedRevisions, rev) } for _, contract := range c.contracts { data.Contracts = append(data.Contracts, contract) } for oldID, newID := range c.renewedIDs { data.RenewedIDs[oldID.String()] = newID.String() } return data } // load loads the Contractor persistence data from disk. func (c *Contractor) load() error { var data contractorPersist err := c.persist.load(&data) if err != nil { return err } c.allowance = data.Allowance c.blockHeight = data.BlockHeight for _, rev := range data.CachedRevisions { c.cachedRevisions[rev.revision.ParentID] = rev } for _, contract := range data.Contracts { c.contracts[contract.ID] = contract } c.financialMetrics = data.FinancialMetrics c.lastChange = data.LastChange for oldString, newString := range data.RenewedIDs { var oldHash, newHash crypto.Hash oldHash.LoadString(oldString) newHash.LoadString(newString) c.renewedIDs[types.FileContractID(oldHash)] = types.FileContractID(newHash) } return nil } // save saves the Contractor persistence data to disk. func (c *Contractor) save() error { return c.persist.save(c.persistData()) } // saveSync saves the Contractor persistence data to disk and then syncs to disk. func (c *Contractor) saveSync() error { return c.persist.saveSync(c.persistData()) } // saveRevision returns a function that saves a revision. It is used by the // Editor and Downloader types to prevent desynchronizing with their host. func (c *Contractor) saveRevision(id types.FileContractID) func(types.FileContractRevision, []crypto.Hash) error { return func(rev types.FileContractRevision, newRoots []crypto.Hash) error { c.mu.Lock() defer c.mu.Unlock() c.cachedRevisions[id] = cachedRevision{rev, newRoots} return c.saveSync() } } Sia-1.0.4/modules/renter/contractor/persist_test.go000066400000000000000000000051131302033070500224250ustar00rootroot00000000000000package contractor import ( "os" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // memPersist implements the persister interface in-memory. type memPersist contractorPersist func (m *memPersist) save(data contractorPersist) error { *m = memPersist(data); return nil } func (m *memPersist) saveSync(data contractorPersist) error { *m = memPersist(data); return nil } func (m memPersist) load(data *contractorPersist) error { *data = contractorPersist(m); return nil } // TestSaveLoad tests that the contractor can save and load itself. func TestSaveLoad(t *testing.T) { // create contractor with mocked persist dependency c := &Contractor{ contracts: make(map[types.FileContractID]modules.RenterContract), renewedIDs: make(map[types.FileContractID]types.FileContractID), } c.persist = new(memPersist) // add some fake contracts c.contracts = map[types.FileContractID]modules.RenterContract{ {0}: {NetAddress: "foo"}, {1}: {NetAddress: "bar"}, {2}: {NetAddress: "baz"}, } c.renewedIDs = map[types.FileContractID]types.FileContractID{ {0}: {1}, {1}: {2}, {2}: {3}, } // save and reload err := c.save() if err != nil { t.Fatal(err) } err = c.load() if err != nil { t.Fatal(err) } // check that contracts were restored _, ok0 := c.contracts[types.FileContractID{0}] _, ok1 := c.contracts[types.FileContractID{1}] _, ok2 := c.contracts[types.FileContractID{2}] if !ok0 || !ok1 || !ok2 { t.Fatal("contracts were not restored properly:", c.contracts) } _, ok0 = c.renewedIDs[types.FileContractID{0}] _, ok1 = c.renewedIDs[types.FileContractID{1}] _, ok2 = c.renewedIDs[types.FileContractID{2}] if !ok0 || !ok1 || !ok2 { t.Fatal("renewed IDs were not restored properly:", c.renewedIDs) } // use stdPersist instead of mock c.persist = newPersist(build.TempDir("contractor", "TestSaveLoad")) os.MkdirAll(build.TempDir("contractor", "TestSaveLoad"), 0700) // save and reload err = c.save() if err != nil { t.Fatal(err) } err = c.load() if err != nil { t.Fatal(err) } // check that contracts were restored _, ok0 = c.contracts[types.FileContractID{0}] _, ok1 = c.contracts[types.FileContractID{1}] _, ok2 = c.contracts[types.FileContractID{2}] if !ok0 || !ok1 || !ok2 { t.Fatal("contracts were not restored properly:", c.contracts) } _, ok0 = c.renewedIDs[types.FileContractID{0}] _, ok1 = c.renewedIDs[types.FileContractID{1}] _, ok2 = c.renewedIDs[types.FileContractID{2}] if !ok0 || !ok1 || !ok2 { t.Fatal("renewed IDs were not restored properly:", c.renewedIDs) } } Sia-1.0.4/modules/renter/contractor/renew.go000066400000000000000000000100251302033070500210130ustar00rootroot00000000000000package contractor import ( "errors" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/renter/proto" "github.com/NebulousLabs/Sia/types" ) // managedRenew negotiates a new contract for data already stored with a host. // It returns the new contract. This is a blocking call that // performs network I/O. func (c *Contractor) managedRenew(contract modules.RenterContract, numSectors uint64, newEndHeight types.BlockHeight) (modules.RenterContract, error) { host, ok := c.hdb.Host(contract.NetAddress) if !ok { return modules.RenterContract{}, errors.New("no record of that host") } else if host.StoragePrice.Cmp(maxStoragePrice) > 0 { return modules.RenterContract{}, errTooExpensive } // cap host.MaxCollateral if host.MaxCollateral.Cmp(maxCollateral) > 0 { host.MaxCollateral = maxCollateral } // get an address to use for negotiation uc, err := c.wallet.NextAddress() if err != nil { return modules.RenterContract{}, err } // create contract params c.mu.RLock() params := proto.ContractParams{ Host: host, Filesize: numSectors * modules.SectorSize, StartHeight: c.blockHeight, EndHeight: newEndHeight, RefundAddress: uc.UnlockHash(), } c.mu.RUnlock() txnBuilder := c.wallet.StartTransaction() // execute negotiation protocol newContract, err := proto.Renew(contract, params, txnBuilder, c.tpool) if err != nil { txnBuilder.Drop() // return unused outputs to wallet return modules.RenterContract{}, err } return newContract, nil } // managedRenewContracts renews any contracts that are up for renewal, using // the current allowance. func (c *Contractor) managedRenewContracts() error { c.mu.RLock() // Renew contracts when they enter the renew window. var renewSet []types.FileContractID for _, contract := range c.contracts { if c.blockHeight+c.allowance.RenewWindow >= contract.EndHeight() { renewSet = append(renewSet, contract.ID) } } c.mu.RUnlock() if len(renewSet) == 0 { // nothing to do return nil } c.log.Printf("renewing %v contracts", len(renewSet)) c.mu.RLock() endHeight := c.blockHeight + c.allowance.Period numSectors, err := maxSectors(c.allowance, c.hdb, c.tpool) c.mu.RUnlock() if err != nil { return err } else if numSectors == 0 { return errors.New("allowance is too small") } // invalidate all active editors/downloaders for the contracts we want to // renew c.mu.Lock() for _, id := range renewSet { c.renewing[id] = true } c.mu.Unlock() // after we finish renewing, unset the 'renewing' flag on each contract defer func() { c.mu.Lock() for _, id := range renewSet { delete(c.renewing, id) } c.mu.Unlock() }() // wait for all active editors and downloaders to finish, then grab the // latest revision of each contract var oldContracts []modules.RenterContract for _, id := range renewSet { c.mu.RLock() e, eok := c.editors[id] d, dok := c.downloaders[id] c.mu.RUnlock() if eok { e.invalidate() } if dok { d.invalidate() } c.mu.RLock() contract, ok := c.contracts[id] c.mu.RUnlock() if !ok { c.log.Printf("WARN: no record of contract previously added to the renew set (ID: %v)", id) continue } oldContracts = append(oldContracts, contract) } // map old ID to new contract, for easy replacement later newContracts := make(map[types.FileContractID]modules.RenterContract) for _, contract := range oldContracts { newContract, err := c.managedRenew(contract, numSectors, endHeight) if err != nil { c.log.Printf("WARN: failed to renew contract with %v: %v", contract.NetAddress, err) } else { newContracts[contract.ID] = newContract } if build.Release != "testing" { // sleep for 1 minute to alleviate potential block propagation issues time.Sleep(60 * time.Second) } } // replace old contracts with renewed ones c.mu.Lock() for id, contract := range newContracts { delete(c.contracts, id) c.contracts[contract.ID] = contract c.renewedIDs[id] = contract.ID } err = c.saveSync() c.mu.Unlock() return err } Sia-1.0.4/modules/renter/contractor/update.go000066400000000000000000000043551302033070500211660ustar00rootroot00000000000000package contractor import ( "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // ProcessConsensusChange will be called by the consensus set every time there // is a change in the blockchain. Updates will always be called in order. func (c *Contractor) ProcessConsensusChange(cc modules.ConsensusChange) { c.mu.Lock() for _, block := range cc.RevertedBlocks { if block.ID() != types.GenesisID { c.blockHeight-- } } for _, block := range cc.AppliedBlocks { if block.ID() != types.GenesisID { c.blockHeight++ } } // delete expired contracts var expired []types.FileContractID for id, contract := range c.contracts { if c.blockHeight > contract.EndHeight() { // No need to wait for extra confirmations - any processes which // depend on this contract should have taken care of any issues // already. expired = append(expired, id) } } for _, id := range expired { delete(c.contracts, id) c.log.Debugln("INFO: deleted expired contract", id) } c.lastChange = cc.ID err := c.save() if err != nil { c.log.Println("Unable to save while processing a consensus change:", err) } c.mu.Unlock() // only attempt contract formation/renewal if we are synced // (harmless if not synced, since hosts will reject our renewal attempts, // (but very slow) if cc.Synced { go func() { // only one goroutine should be editing contracts at a time if !c.editLock.TryLock() { return } defer c.editLock.Unlock() // renew any contracts that have entered the renew window err := c.managedRenewContracts() if err != nil { c.log.Debugln("WARN: failed to renew contracts after processing a consensus chage:", err) } // if we don't have enough contracts, form new ones c.mu.RLock() a := c.allowance remaining := int(a.Hosts) - len(c.contracts) c.mu.RUnlock() if remaining <= 0 { return } numSectors, err := maxSectors(a, c.hdb, c.tpool) if err != nil { c.log.Debugln("ERROR: couldn't calculate maxSectors after processing a consensus change:", err) return } err = c.managedFormAllowanceContracts(remaining, numSectors, a) if err != nil { c.log.Debugln("WARN: failed to form contracts after processing a consensus change:", err) } }() } } Sia-1.0.4/modules/renter/contractor/update_test.go000066400000000000000000000132261302033070500222220ustar00rootroot00000000000000package contractor import ( "io/ioutil" "testing" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/types" ) // TestProcessConsensusUpdate tests that contracts are removed at the expected // block height. func TestProcessConsensusUpdate(t *testing.T) { // create contractor with a contract ending at height 20 var stub newStub var rc modules.RenterContract rc.LastRevision.NewWindowStart = 20 rc.FileContract.ValidProofOutputs = []types.SiacoinOutput{{}} c := &Contractor{ cs: stub, hdb: stub, contracts: map[types.FileContractID]modules.RenterContract{ rc.ID: rc, }, persist: new(memPersist), log: persist.NewLogger(ioutil.Discard), } // process 20 blocks; contract should remain cc := modules.ConsensusChange{ // just need to increment blockheight by 1 AppliedBlocks: []types.Block{{}}, } for i := 0; i < 20; i++ { c.ProcessConsensusChange(cc) } if len(c.contracts) != 1 { t.Error("expected 1 contract, got", len(c.contracts)) } // process one more block; contract should be removed c.ProcessConsensusChange(cc) if len(c.contracts) != 0 { t.Error("expected 0 contracts, got", len(c.contracts)) } } // TestIntegrationAutoRenew tests that contracts are automatically renwed at // the expected block height. func TestIntegrationAutoRenew(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // create testing trio _, c, m, err := newTestingTrio("TestIntegrationAutoRenew") if err != nil { t.Fatal(err) } // form a contract with the host a := modules.Allowance{ Funds: types.SiacoinPrecision.Mul64(100), // 100 SC Hosts: 1, Period: 50, RenewWindow: 10, } err = c.SetAllowance(a) if err != nil { t.Fatal(err) } contract := c.Contracts()[0] // revise the contract editor, err := c.Editor(contract.ID) if err != nil { t.Fatal(err) } data, err := crypto.RandBytes(int(modules.SectorSize)) if err != nil { t.Fatal(err) } // insert the sector root, err := editor.Upload(data) if err != nil { t.Fatal(err) } err = editor.Close() if err != nil { t.Fatal(err) } // mine until we enter the renew window renewHeight := contract.EndHeight() - c.allowance.RenewWindow for c.blockHeight < renewHeight { _, err := m.AddBlock() if err != nil { t.Fatal(err) } } // wait for goroutine in ProcessConsensusChange to finish time.Sleep(100 * time.Millisecond) c.editLock.Lock() c.editLock.Unlock() // check renewed contract contract = c.Contracts()[0] if contract.FileContract.FileMerkleRoot != root { t.Fatal("wrong merkle root:", contract.FileContract.FileMerkleRoot) } else if contract.FileContract.FileSize != modules.SectorSize { t.Fatal("wrong file size:", contract.FileContract.FileSize) } else if contract.FileContract.RevisionNumber != 0 { t.Fatal("wrong revision number:", contract.FileContract.RevisionNumber) } else if contract.FileContract.WindowStart != c.blockHeight+c.allowance.Period { t.Fatal("wrong window start:", contract.FileContract.WindowStart) } } // TestIntegrationRenewInvalidate tests that editors and downloaders are // properly invalidated when a renew is queued. func TestIntegrationRenewInvalidate(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() // create testing trio _, c, m, err := newTestingTrio("TestIntegrationRenewInvalidate") if err != nil { t.Fatal(err) } // form a contract with the host a := modules.Allowance{ Funds: types.SiacoinPrecision.Mul64(100), // 100 SC Hosts: 1, Period: 50, RenewWindow: 10, } err = c.SetAllowance(a) if err != nil { t.Fatal(err) } contract := c.Contracts()[0] // revise the contract editor, err := c.Editor(contract.ID) if err != nil { t.Fatal(err) } data, err := crypto.RandBytes(int(modules.SectorSize)) if err != nil { t.Fatal(err) } // insert the sector root, err := editor.Upload(data) if err != nil { t.Fatal(err) } // mine until we enter the renew window renewHeight := contract.EndHeight() - c.allowance.RenewWindow for c.blockHeight < renewHeight { _, err := m.AddBlock() if err != nil { t.Fatal(err) } } // wait for goroutine in ProcessConsensusChange to finish time.Sleep(100 * time.Millisecond) c.editLock.Lock() c.editLock.Unlock() // check renewed contract contract = c.Contracts()[0] if contract.FileContract.FileMerkleRoot != root { t.Error("wrong merkle root:", contract.FileContract.FileMerkleRoot) } else if contract.FileContract.FileSize != modules.SectorSize { t.Error("wrong file size:", contract.FileContract.FileSize) } else if contract.FileContract.RevisionNumber != 0 { t.Error("wrong revision number:", contract.FileContract.RevisionNumber) } else if contract.FileContract.WindowStart != c.blockHeight+c.allowance.Period { t.Error("wrong window start:", contract.FileContract.WindowStart) } // editor should have been invalidated err = editor.Delete(crypto.Hash{}) if err != errInvalidEditor { t.Error("expected invalid editor error; got", err) } editor.Close() // create a downloader downloader, err := c.Downloader(contract.ID) if err != nil { t.Fatal(err) } // mine until we enter the renew window renewHeight = contract.EndHeight() - c.allowance.RenewWindow for c.blockHeight < renewHeight { _, err := m.AddBlock() if err != nil { t.Fatal(err) } } // wait for goroutine in ProcessConsensusChange to finish time.Sleep(100 * time.Millisecond) c.editLock.Lock() c.editLock.Unlock() // downloader should have been invalidated _, err = downloader.Sector(crypto.Hash{}) if err != errInvalidDownloader { t.Error("expected invalid downloader error; got", err) } downloader.Close() } Sia-1.0.4/modules/renter/contractor/upload_test.go000066400000000000000000000042611302033070500222230ustar00rootroot00000000000000package contractor import ( "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // editorHostDB is used to test the Editor method. type editorHostDB struct { stubHostDB hosts map[modules.NetAddress]modules.HostDBEntry } func (hdb editorHostDB) Host(addr modules.NetAddress) (modules.HostDBEntry, bool) { h, ok := hdb.hosts[addr] return h, ok } // TestEditor tests the failure conditions of the Editor method. The method is // more fully tested in the host integration test. func TestEditor(t *testing.T) { // use a mock hostdb to supply hosts hdb := &editorHostDB{ hosts: make(map[modules.NetAddress]modules.HostDBEntry), } c := &Contractor{ hdb: hdb, revising: make(map[types.FileContractID]bool), contracts: make(map[types.FileContractID]modules.RenterContract), } // empty contract ID _, err := c.Editor(types.FileContractID{}) if err == nil { t.Error("expected error, got nil") } // expired contract c.blockHeight = 3 _, err = c.Editor(types.FileContractID{}) if err == nil { t.Error("expected error, got nil") } c.blockHeight = 0 // expensive host _, hostPublicKey := crypto.GenerateKeyPairDeterministic([32]byte{}) dbe := modules.HostDBEntry{ PublicKey: types.SiaPublicKey{ Algorithm: types.SignatureEd25519, Key: hostPublicKey[:], }, } dbe.AcceptingContracts = true dbe.StoragePrice = types.NewCurrency64(^uint64(0)) hdb.hosts["foo"] = dbe contract := modules.RenterContract{NetAddress: "foo"} c.contracts[contract.ID] = contract _, err = c.Editor(contract.ID) if err == nil { t.Error("expected error, got nil") } // invalid contract dbe.StoragePrice = types.NewCurrency64(500) hdb.hosts["bar"] = dbe _, err = c.Editor(contract.ID) if err == nil { t.Error("expected error, got nil") } // spent contract c.contracts[contract.ID] = modules.RenterContract{ NetAddress: "bar", LastRevision: types.FileContractRevision{ NewValidProofOutputs: []types.SiacoinOutput{ {Value: types.NewCurrency64(0)}, {Value: types.NewCurrency64(^uint64(0))}, }, }, } _, err = c.Editor(contract.ID) if err == nil { t.Error("expected error, got nil") } } Sia-1.0.4/modules/renter/download.go000066400000000000000000000206311302033070500173300ustar00rootroot00000000000000package renter import ( "errors" "fmt" "io" "os" "strings" "sync/atomic" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/renter/contractor" ) var ( errInsufficientHosts = errors.New("insufficient hosts to recover file") errInsufficientPieces = errors.New("couldn't fetch enough pieces to recover data") ) // A fetcher fetches pieces from a host. This interface exists to facilitate // easy testing. type fetcher interface { // pieces returns the set of pieces corresponding to a given chunk. pieces(chunk uint64) []pieceData // fetch returns the data specified by piece metadata. fetch(pieceData) ([]byte, error) } // A hostFetcher fetches pieces from a host. It implements the fetcher // interface. type hostFetcher struct { downloader contractor.Downloader pieceMap map[uint64][]pieceData masterKey crypto.TwofishKey } // pieces returns the pieces stored on this host that are part of a given // chunk. func (hf *hostFetcher) pieces(chunk uint64) []pieceData { return hf.pieceMap[chunk] } // fetch downloads the piece specified by p. func (hf *hostFetcher) fetch(p pieceData) ([]byte, error) { // request piece data, err := hf.downloader.Sector(p.MerkleRoot) if err != nil { return nil, err } // generate decryption key key := deriveKey(hf.masterKey, p.Chunk, p.Piece) // decrypt and return return key.DecryptBytes(data) } // newHostFetcher creates a new hostFetcher. func newHostFetcher(d contractor.Downloader, pieces []pieceData, masterKey crypto.TwofishKey) *hostFetcher { // make piece map pieceMap := make(map[uint64][]pieceData) for _, p := range pieces { pieceMap[p.Chunk] = append(pieceMap[p.Chunk], p) } return &hostFetcher{ downloader: d, pieceMap: pieceMap, masterKey: masterKey, } } // checkHosts checks that a set of hosts is sufficient to download a file. func checkHosts(hosts []fetcher, minPieces int, numChunks uint64) error { for i := uint64(0); i < numChunks; i++ { pieces := 0 for _, h := range hosts { pieces += len(h.pieces(i)) } if pieces < minPieces { return errInsufficientHosts } } return nil } // A download is a file download that has been queued by the renter. type download struct { // NOTE: received is the first field to ensure 64-bit alignment, which is // required for atomic operations. received uint64 chunkIndex uint64 startTime time.Time siapath string destination string erasureCode modules.ErasureCoder chunkSize uint64 fileSize uint64 hosts []fetcher } // getPiece locates and downloads a specific piece. func (d *download) getPiece(chunkIndex, pieceIndex uint64) []byte { for _, h := range d.hosts { for _, p := range h.pieces(chunkIndex) { if p.Piece == pieceIndex { data, err := h.fetch(p) if err != nil { break // try next host } return data } } } return nil } // run performs the actual download. It spawns one worker per host, and // instructs them to sequentially download chunks. It then writes the recovered // chunks to w. It returns its progress along with a bool indicating whether // another iteration should be used. func (d *download) run(w io.Writer) error { for ; d.received < d.fileSize; d.chunkIndex++ { // load pieces into chunk chunk := make([][]byte, d.erasureCode.NumPieces()) left := d.erasureCode.MinPieces() // pick hosts at random chunkOrder, err := crypto.Perm(len(chunk)) if err != nil { return err } for _, j := range chunkOrder { chunk[j] = d.getPiece(d.chunkIndex, uint64(j)) if chunk[j] != nil { left-- } if left == 0 { break } } if left != 0 { return errInsufficientPieces } // Write pieces to w. We always write chunkSize bytes unless this is // the last chunk; in that case, we write the remainder. n := d.chunkSize if n > d.fileSize-d.received { n = d.fileSize - d.received } err = d.erasureCode.Recover(chunk, uint64(n), w) if err != nil { return err } atomic.AddUint64(&d.received, n) } return nil } // newDownload initializes and returns a download object. func (f *file) newDownload(hosts []fetcher, destination string) *download { d := &download{ erasureCode: f.erasureCode, chunkSize: f.chunkSize(), fileSize: f.size, hosts: hosts, startTime: time.Now(), chunkIndex: 0, received: 0, siapath: f.name, destination: destination, } return d } // Download downloads a file, identified by its path, to the destination // specified. func (r *Renter) Download(path, destination string) error { // Lookup the file associated with the nickname. lockID := r.mu.RLock() file, exists := r.files[path] r.mu.RUnlock(lockID) if !exists { return errors.New("no file with that path") } // Create the download object and add it to the queue. d := file.newDownload([]fetcher{}, destination) lockID = r.mu.Lock() r.downloadQueue = append(r.downloadQueue, d) r.mu.Unlock(lockID) // Create file on disk with the correct permissions. perm := os.FileMode(file.mode) if perm == 0 { // sane default perm = 0666 } f, err := os.OpenFile(destination, os.O_CREATE|os.O_RDWR|os.O_TRUNC, perm) if err != nil { return err } defer f.Close() // A loop that will iterate until the download is complete. // Downloads are canceled if they make no progress for 120 minutes. progressDeadline := time.Now().Add(120 * time.Minute) for { // copy file contracts file.mu.RLock() contracts := make([]fileContract, 0, len(file.contracts)) for _, c := range file.contracts { contracts = append(contracts, c) } file.mu.RUnlock() if len(contracts) < file.erasureCode.MinPieces() { return fmt.Errorf("contracts could not be located for this file - file may not be recoverable - needed %v, got %v", file.erasureCode.MinPieces(), len(contracts)) } // interrupt upload loop lockID = r.mu.Lock() r.downloading = true uploading := r.uploading r.mu.Unlock(lockID) resumeUploads := func() { lockID = r.mu.Lock() r.downloading = false r.mu.Unlock(lockID) } // wait up to 60 minutes for upload loop to exit timeout := time.Now().Add(60 * time.Minute) for uploading && time.Now().Before(timeout) { time.Sleep(time.Second) lockID = r.mu.RLock() uploading = r.uploading r.mu.RUnlock(lockID) } if uploading { resumeUploads() return errors.New("timed out waiting for uploads to finish") } // Grab a set of hosts and attempt a download. done, err := func() (bool, error) { // Initiate connections to each host. var hosts []fetcher var errs []string for _, c := range file.contracts { d, err := r.hostContractor.Downloader(c.ID) if err != nil { errs = append(errs, fmt.Sprintf("\t%v: %v", c.IP, err)) continue } defer d.Close() hosts = append(hosts, newHostFetcher(d, c.Pieces, file.masterKey)) } if len(hosts) < file.erasureCode.MinPieces() { return false, errors.New("could not connect to enough hosts:\n" + strings.Join(errs, "\n")) } // Check that this host set is sufficient to download the file. err := checkHosts(hosts, file.erasureCode.MinPieces(), file.numChunks()) if err != nil { return false, err } // Update the downloader with the new set of hosts. d.hosts = hosts // Perform download. err = d.run(f) done := err == nil return done, nil }() if done { // Download is complete! resumeUploads() break } else if err != nil { // One of the more severe errors occurred, wait a bit before trying // the download again. resumeUploads() time.Sleep(time.Second * 90) } else { // We made progress, but haven't finished yet. Reset the progress // deadline. progressDeadline = time.Now().Add(30 * time.Minute) } // if we haven't made any progress in 30 minutes, give up if time.Now().After(progressDeadline) { return errors.New("no progress in 30 minutes; giving up") } } return nil } // DownloadQueue returns the list of downloads in the queue. func (r *Renter) DownloadQueue() []modules.DownloadInfo { lockID := r.mu.RLock() defer r.mu.RUnlock(lockID) // order from most recent to least recent downloads := make([]modules.DownloadInfo, len(r.downloadQueue)) for i := range r.downloadQueue { d := r.downloadQueue[len(r.downloadQueue)-i-1] downloads[i] = modules.DownloadInfo{ SiaPath: d.siapath, Destination: d.destination, Filesize: d.fileSize, Received: atomic.LoadUint64(&d.received), StartTime: d.startTime, } } return downloads } Sia-1.0.4/modules/renter/download_test.go000066400000000000000000000110351302033070500203650ustar00rootroot00000000000000package renter import ( "bytes" "io" "testing" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/renter/contractor" "github.com/NebulousLabs/Sia/types" ) // a testFetcher simulates a host. It implements the fetcher interface. type testFetcher struct { sectors map[crypto.Hash][]byte pieceMap map[uint64][]pieceData pieceSize uint64 nAttempt int // total number of download attempts nFetch int // number of successful download attempts // used to simulate real-world conditions delay time.Duration // transfers will take this long failRate int // transfers will randomly fail with probability 1/failRate } func (f *testFetcher) pieces(chunkIndex uint64) []pieceData { return f.pieceMap[chunkIndex] } func (f *testFetcher) fetch(p pieceData) ([]byte, error) { f.nAttempt++ time.Sleep(f.delay) // randomly fail if n, _ := crypto.RandIntn(f.failRate); n == 0 { return nil, io.EOF } f.nFetch++ return f.sectors[p.MerkleRoot], nil } // TestErasureDownload tests parallel downloading of erasure-coded data. It // mocks the fetcher interface in order to directly test the downloading // algorithm. func TestErasureDownload(t *testing.T) { if testing.Short() { t.SkipNow() } // generate data const dataSize = 777 data, err := crypto.RandBytes(dataSize) if err != nil { t.Fatal(err) } // create Reed-Solomon encoder rsc, err := NewRSCode(2, 10) if err != nil { t.Fatal(err) } // create hosts const pieceSize = 10 hosts := make([]fetcher, rsc.NumPieces()) for i := range hosts { hosts[i] = &testFetcher{ sectors: make(map[crypto.Hash][]byte), pieceMap: make(map[uint64][]pieceData), pieceSize: pieceSize, delay: time.Millisecond, failRate: 5, // 20% failure rate } } // make one host really slow hosts[0].(*testFetcher).delay = 100 * time.Millisecond // make one host always fail hosts[1].(*testFetcher).failRate = 1 // upload data to hosts r := bytes.NewReader(data) // makes chunking easier chunk := make([]byte, pieceSize*rsc.MinPieces()) var i uint64 for i = uint64(0); ; i++ { _, err := io.ReadFull(r, chunk) if err == io.EOF { break } else if err != nil && err != io.ErrUnexpectedEOF { t.Fatal(err) } pieces, err := rsc.Encode(chunk) if err != nil { t.Fatal(err) } for j, p := range pieces { root := crypto.MerkleRoot(p) host := hosts[j%len(hosts)].(*testFetcher) // distribute evenly host.pieceMap[i] = append(host.pieceMap[i], pieceData{ Chunk: uint64(i), Piece: uint64(j), MerkleRoot: root, }) host.sectors[root] = p } } // check hosts (not strictly necessary) err = checkHosts(hosts, rsc.MinPieces(), i) if err != nil { t.Fatal(err) } // download data d := newFile("foo", rsc, pieceSize, dataSize).newDownload(hosts, "") buf := new(bytes.Buffer) err = d.run(buf) if err != nil { t.Fatal(err) } if !bytes.Equal(buf.Bytes(), data) { t.Fatal("recovered data does not match original") } /* // These metrics can be used to assess the efficiency of the download // algorithm. totFetch := 0 for i, h := range hosts { h := h.(*testHost) t.Logf("Host %2d: Fetched: %v/%v", i, h.nFetch, h.nAttempt) totFetch += h.nAttempt } t.Log("Optimal fetches:", i*uint64(rsc.MinPieces())) t.Log("Total fetches: ", totFetch) */ } type downloadContractor struct { stubContractor downloaders int } func (dc *downloadContractor) Contract(modules.NetAddress) (modules.RenterContract, bool) { return modules.RenterContract{}, true } // Downloader increments dc.downloaders and returns a generic error. func (dc *downloadContractor) Downloader(types.FileContractID) (contractor.Downloader, error) { dc.downloaders++ return nil, errInsufficientContracts } // TestDownloadContracts tests that Download is properly creating Downloaders // for each contract. func TestDownloadContracts(t *testing.T) { t.Skip("incompatible with new download loop") if testing.Short() { t.SkipNow() } var hc downloadContractor rt, err := newContractorTester("TestDownloadContracts", nil, &hc) if err != nil { t.Fatal(err) } // add a fake file rsc, _ := NewRSCode(1, 1) f := newFile("foo", rsc, 0, 0) const nContracts = 10 for i := byte(0); i < nContracts; i++ { f.contracts[types.FileContractID{i}] = fileContract{} } id := rt.renter.mu.Lock() rt.renter.files["foo"] = f rt.renter.mu.Unlock(id) rt.renter.Download("foo", "") if hc.downloaders != nContracts { t.Fatalf("expected Downloader to be called %v times, got %v", nContracts, hc.downloaders) } } Sia-1.0.4/modules/renter/erasure.go000066400000000000000000000034261302033070500171720ustar00rootroot00000000000000package renter import ( "io" "github.com/klauspost/reedsolomon" "github.com/NebulousLabs/Sia/modules" ) // rsCode is a Reed-Solomon encoder/decoder. It implements the // modules.ErasureCoder interface. type rsCode struct { enc reedsolomon.Encoder numPieces int dataPieces int } // NumPieces returns the number of pieces returned by Encode. func (rs *rsCode) NumPieces() int { return rs.numPieces } // MinPieces return the minimum number of pieces that must be present to // recover the original data. func (rs *rsCode) MinPieces() int { return rs.dataPieces } // Encode splits data into equal-length pieces, some containing the original // data and some containing parity data. func (rs *rsCode) Encode(data []byte) ([][]byte, error) { pieces, err := rs.enc.Split(data) if err != nil { return nil, err } // err should not be possible if Encode is called on the result of Split, // but no harm in checking anyway. err = rs.enc.Encode(pieces) if err != nil { return nil, err } return pieces, nil } // Recover recovers the original data from pieces (including parity) and // writes it to w. pieces should be identical to the slice returned by // Encode (length and order must be preserved), but with missing elements // set to nil. func (rs *rsCode) Recover(pieces [][]byte, n uint64, w io.Writer) error { err := rs.enc.Reconstruct(pieces) if err != nil { return err } // TODO: implement this manually return rs.enc.Join(w, pieces, int(n)) } // NewRSCode creates a new Reed-Solomon encoder/decoder using the supplied // parameters. func NewRSCode(nData, nParity int) (modules.ErasureCoder, error) { enc, err := reedsolomon.New(nData, nParity) if err != nil { return nil, err } return &rsCode{ enc: enc, numPieces: nData + nParity, dataPieces: nData, }, nil } Sia-1.0.4/modules/renter/erasure_test.go000066400000000000000000000030551302033070500202270ustar00rootroot00000000000000package renter import ( "bytes" "crypto/rand" "io/ioutil" "testing" ) // TestRSEncode tests the rsCode type. func TestRSEncode(t *testing.T) { badParams := []struct { data, parity int }{ {-1, -1}, {-1, 0}, {0, -1}, {0, 0}, {0, 1}, {1, 0}, } for _, ps := range badParams { if _, err := NewRSCode(ps.data, ps.parity); err == nil { t.Error("expected bad parameter error, got nil") } } rsc, err := NewRSCode(10, 3) if err != nil { t.Fatal(err) } data := make([]byte, 777) rand.Read(data) pieces, err := rsc.Encode(data) if err != nil { t.Fatal(err) } _, err = rsc.Encode(nil) if err == nil { t.Fatal("expected nil data error, got nil") } buf := new(bytes.Buffer) err = rsc.Recover(pieces, 777, buf) if err != nil { t.Fatal(err) } err = rsc.Recover(nil, 777, buf) if err == nil { t.Fatal("expected nil pieces error, got nil") } if !bytes.Equal(data, buf.Bytes()) { t.Fatal("recovered data does not match original") } } func BenchmarkRSEncode(b *testing.B) { rsc, err := NewRSCode(80, 20) if err != nil { b.Fatal(err) } data := make([]byte, 1<<20) rand.Read(data) b.SetBytes(1 << 20) b.ResetTimer() for i := 0; i < b.N; i++ { rsc.Encode(data) } } func BenchmarkRSRecover(b *testing.B) { rsc, err := NewRSCode(50, 200) if err != nil { b.Fatal(err) } data := make([]byte, 1<<20) rand.Read(data) pieces, err := rsc.Encode(data) if err != nil { b.Fatal(err) } b.SetBytes(1 << 20) b.ResetTimer() for i := 0; i < b.N; i++ { pieces[0] = nil rsc.Recover(pieces, 1<<20, ioutil.Discard) } } Sia-1.0.4/modules/renter/files.go000066400000000000000000000173351302033070500166320ustar00rootroot00000000000000package renter import ( "errors" "os" "path/filepath" "sync" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( ErrEmptyFilename = errors.New("filename must be a nonempty string") ErrUnknownPath = errors.New("no file known with that path") ErrPathOverload = errors.New("a file already exists at that location") ) // A file is a single file that has been uploaded to the network. Files are // split into equal-length chunks, which are then erasure-coded into pieces. // Each piece is separately encrypted, using a key derived from the file's // master key. The pieces are uploaded to hosts in groups, such that one file // contract covers many pieces. type file struct { name string size uint64 contracts map[types.FileContractID]fileContract masterKey crypto.TwofishKey erasureCode modules.ErasureCoder pieceSize uint64 mode uint32 // actually an os.FileMode mu sync.RWMutex } // A fileContract is a contract covering an arbitrary number of file pieces. // Chunk/Piece metadata is used to split the raw contract data appropriately. type fileContract struct { ID types.FileContractID IP modules.NetAddress Pieces []pieceData WindowStart types.BlockHeight } // pieceData contains the metadata necessary to request a piece from a // fetcher. type pieceData struct { Chunk uint64 // which chunk the piece belongs to Piece uint64 // the index of the piece in the chunk MerkleRoot crypto.Hash // the Merkle root of the piece } // deriveKey derives the key used to encrypt and decrypt a specific file piece. func deriveKey(masterKey crypto.TwofishKey, chunkIndex, pieceIndex uint64) crypto.TwofishKey { return crypto.TwofishKey(crypto.HashAll(masterKey, chunkIndex, pieceIndex)) } // chunkSize returns the size of one chunk. func (f *file) chunkSize() uint64 { return f.pieceSize * uint64(f.erasureCode.MinPieces()) } // numChunks returns the number of chunks that f was split into. func (f *file) numChunks() uint64 { // empty files still need at least one chunk if f.size == 0 { return 1 } n := f.size / f.chunkSize() // last chunk will be padded, unless chunkSize divides file evenly. if f.size%f.chunkSize() != 0 { n++ } return n } // available indicates whether the file is ready to be downloaded. func (f *file) available() bool { f.mu.RLock() defer f.mu.RUnlock() chunkPieces := make([]int, f.numChunks()) for _, fc := range f.contracts { for _, p := range fc.Pieces { chunkPieces[p.Chunk]++ } } for _, n := range chunkPieces { if n < f.erasureCode.MinPieces() { return false } } return true } // uploadProgress indicates what percentage of the file (plus redundancy) has // been uploaded. Note that a file may be Available long before UploadProgress // reaches 100%, and UploadProgress may report a value greater than 100%. func (f *file) uploadProgress() float64 { f.mu.RLock() defer f.mu.RUnlock() var uploaded uint64 for _, fc := range f.contracts { uploaded += uint64(len(fc.Pieces)) * f.pieceSize } desired := f.pieceSize * uint64(f.erasureCode.NumPieces()) * f.numChunks() return 100 * (float64(uploaded) / float64(desired)) } // redundancy returns the redundancy of the least redundant chunk. A file // becomes available when this redundancy is >= 1. Assumes that every piece is // unique within a file contract. -1 is returned if the file has size 0. func (f *file) redundancy() float64 { if f.size == 0 { return -1 } piecesPerChunk := make([]int, f.numChunks()) // If the file has non-0 size then the number of chunks should also be // non-0. Therefore the f.size == 0 conditional block above must appear // before this check. if len(piecesPerChunk) == 0 { build.Critical("cannot get redundancy of a file with 0 chunks") return -1 } for _, fc := range f.contracts { for _, p := range fc.Pieces { piecesPerChunk[p.Chunk]++ } } minPieces := piecesPerChunk[0] for _, numPieces := range piecesPerChunk { if numPieces < minPieces { minPieces = numPieces } } return float64(minPieces) / float64(f.erasureCode.MinPieces()) } // expiration returns the lowest height at which any of the file's contracts // will expire. func (f *file) expiration() types.BlockHeight { f.mu.RLock() defer f.mu.RUnlock() if len(f.contracts) == 0 { return 0 } lowest := ^types.BlockHeight(0) for _, fc := range f.contracts { if fc.WindowStart < lowest { lowest = fc.WindowStart } } return lowest } // newFile creates a new file object. func newFile(name string, code modules.ErasureCoder, pieceSize, fileSize uint64) *file { key, _ := crypto.GenerateTwofishKey() return &file{ name: name, size: fileSize, contracts: make(map[types.FileContractID]fileContract), masterKey: key, erasureCode: code, pieceSize: pieceSize, } } // DeleteFile removes a file entry from the renter and deletes its data from // the hosts it is stored on. func (r *Renter) DeleteFile(nickname string) error { lockID := r.mu.Lock() f, exists := r.files[nickname] if !exists { r.mu.Unlock(lockID) return ErrUnknownPath } delete(r.files, nickname) os.RemoveAll(filepath.Join(r.persistDir, f.name+ShareExtension)) r.saveSync() r.mu.Unlock(lockID) // delete the file's associated contract data. f.mu.Lock() defer f.mu.Unlock() // TODO: this is ugly because we only have the Contracts method for // looking up contracts. var contracts []modules.RenterContract for _, c := range r.hostContractor.Contracts() { if _, ok := f.contracts[c.ID]; ok { contracts = append(contracts, c) } } for _, c := range contracts { editor, err := r.hostContractor.Editor(c.ID) if err != nil { // TODO: what if the host isn't online? continue } for _, root := range c.MerkleRoots { editor.Delete(root) } delete(f.contracts, c.ID) } return nil } // FileList returns all of the files that the renter has. func (r *Renter) FileList() []modules.FileInfo { lockID := r.mu.RLock() defer r.mu.RUnlock(lockID) files := make([]modules.FileInfo, 0, len(r.files)) for _, f := range r.files { // _, renewing := r.tracking[f.name] // TODO: bring back per-file renewing renewing := true files = append(files, modules.FileInfo{ SiaPath: f.name, Filesize: f.size, Available: f.available(), Redundancy: f.redundancy(), Renewing: renewing, UploadProgress: f.uploadProgress(), Expiration: f.expiration(), }) } return files } // RenameFile takes an existing file and changes the nickname. The original // file must exist, and there must not be any file that already has the // replacement nickname. func (r *Renter) RenameFile(currentName, newName string) error { lockID := r.mu.Lock() defer r.mu.Unlock(lockID) // Check that newName is nonempty. if newName == "" { return ErrEmptyFilename } // Check that currentName exists and newName doesn't. file, exists := r.files[currentName] if !exists { return ErrUnknownPath } _, exists = r.files[newName] if exists { return ErrPathOverload } // Modify the file and save it to disk. file.mu.Lock() file.name = newName err := r.saveFile(file) file.mu.Unlock() if err != nil { return err } // Update the entries in the renter. delete(r.files, currentName) r.files[newName] = file err = r.saveSync() if err != nil { return err } // Delete the old .sia file. // NOTE: proper error handling is difficult here. For example, if the // removal fails, should the entry in r.files be preserved? For now we will // keep things simple, but it is important that our approach feels // intuitive/unsurprising and doesn't put the user's data at risk. oldPath := filepath.Join(r.persistDir, currentName+ShareExtension) return os.RemoveAll(oldPath) } Sia-1.0.4/modules/renter/files_test.go000066400000000000000000000220371302033070500176640ustar00rootroot00000000000000package renter import ( "os" "path/filepath" "testing" "github.com/NebulousLabs/Sia/types" ) // TestFileNumChunks checks the numChunks method of the file type. func TestFileNumChunks(t *testing.T) { tests := []struct { size uint64 pieceSize uint64 piecesPerChunk int expNumChunks uint64 }{ {100, 10, 1, 10}, // evenly divides {100, 10, 2, 5}, // evenly divides {101, 10, 1, 11}, // padded {101, 10, 2, 6}, // padded {10, 100, 1, 1}, // larger piece than file {0, 10, 1, 1}, // 0-length } for _, test := range tests { rsc, _ := NewRSCode(test.piecesPerChunk, 1) // can't use 0 f := &file{size: test.size, erasureCode: rsc, pieceSize: test.pieceSize} if f.numChunks() != test.expNumChunks { t.Errorf("Test %v: expected %v, got %v", test, test.expNumChunks, f.numChunks()) } } } // TestFileAvailable probes the available method of the file type. func TestFileAvailable(t *testing.T) { rsc, _ := NewRSCode(1, 10) f := &file{ size: 1000, erasureCode: rsc, pieceSize: 100, } if f.available() { t.Error("file should not be available") } var fc fileContract for i := uint64(0); i < f.numChunks(); i++ { fc.Pieces = append(fc.Pieces, pieceData{Chunk: i, Piece: 0}) } f.contracts = map[types.FileContractID]fileContract{types.FileContractID{}: fc} if !f.available() { t.Error("file should be available") } } // TestFileRedundancy tests that redundancy is correctly calculated for files // with varying number of filecontracts and erasure code settings. func TestFileRedundancy(t *testing.T) { nDatas := []int{1, 2, 10} for _, nData := range nDatas { rsc, _ := NewRSCode(nData, 10) f := &file{ size: 1000, pieceSize: 100, contracts: make(map[types.FileContractID]fileContract), erasureCode: rsc, } // Test that an empty file has 0 redundancy. if r := f.redundancy(); r != 0 { t.Error("expected 0 redundancy, got", r) } // Test that a file with 1 filecontract that has a piece for every chunk but // one chunk still has a redundancy of 0. fc := fileContract{ ID: types.FileContractID{0}, } for i := uint64(0); i < f.numChunks()-1; i++ { pd := pieceData{ Chunk: i, Piece: 0, } fc.Pieces = append(fc.Pieces, pd) } f.contracts[fc.ID] = fc if r := f.redundancy(); r != 0 { t.Error("expected 0 redundancy, got", r) } // Test that adding another filecontract with a piece for every chunk but one // chunk still results in a file with redundancy 0. fc = fileContract{ ID: types.FileContractID{1}, } for i := uint64(0); i < f.numChunks()-1; i++ { pd := pieceData{ Chunk: i, Piece: 1, } fc.Pieces = append(fc.Pieces, pd) } f.contracts[fc.ID] = fc if r := f.redundancy(); r != 0 { t.Error("expected 0 redundancy, got", r) } // Test that adding a file contract with a piece for the missing chunk // results in a file with redundancy > 0 && <= 1. fc = fileContract{ ID: types.FileContractID{2}, } pd := pieceData{ Chunk: f.numChunks() - 1, Piece: 0, } fc.Pieces = append(fc.Pieces, pd) f.contracts[fc.ID] = fc // 1.0 / MinPieces because the chunk with the least number of pieces has 1 piece. expectedR := 1.0 / float64(f.erasureCode.MinPieces()) if r := f.redundancy(); r == 0 || r > 1 || r != expectedR { t.Errorf("expected %f redundancy, got %f", expectedR, r) } // Test that adding a file contract that has erasureCode.MinPieces() pieces // per chunk for all chunks results in a file with redundancy > 1. fc = fileContract{ ID: types.FileContractID{3}, } for iChunk := uint64(0); iChunk < f.numChunks(); iChunk++ { for iPiece := 0; iPiece < f.erasureCode.MinPieces(); iPiece++ { pd := pieceData{ Chunk: iChunk, Piece: uint64(iPiece), } fc.Pieces = append(fc.Pieces, pd) } } f.contracts[fc.ID] = fc // 1+MinPieces / MinPieces because the chunk with the least number of pieces has 1+MinPieces pieces. expectedR = float64(1+f.erasureCode.MinPieces()) / float64(f.erasureCode.MinPieces()) if r := f.redundancy(); r <= 1 || r != expectedR { t.Errorf("expected a redundancy >1 and equal to %f, got %f", expectedR, r) } } } // TestFileExpiration probes the expiration method of the file type. func TestFileExpiration(t *testing.T) { f := &file{ contracts: make(map[types.FileContractID]fileContract), } if f.expiration() != 0 { t.Error("file with no pieces should report as having no time remaining") } // Add a contract. fc := fileContract{} fc.WindowStart = 100 f.contracts[types.FileContractID{0}] = fc if f.expiration() != 100 { t.Error("file did not report lowest WindowStart") } // Add a contract with a lower WindowStart. fc.WindowStart = 50 f.contracts[types.FileContractID{1}] = fc if f.expiration() != 50 { t.Error("file did not report lowest WindowStart") } // Add a contract with a higher WindowStart. fc.WindowStart = 75 f.contracts[types.FileContractID{2}] = fc if f.expiration() != 50 { t.Error("file did not report lowest WindowStart") } } // TestRenterDeleteFile probes the DeleteFile method of the renter type. func TestRenterDeleteFile(t *testing.T) { if testing.Short() { t.SkipNow() } rt, err := newRenterTester("TestRenterDeleteFile") if err != nil { t.Fatal(err) } defer rt.Close() // Delete a file from an empty renter. err = rt.renter.DeleteFile("dne") if err != ErrUnknownPath { t.Error("Expected ErrUnknownPath:", err) } // Put a file in the renter. rt.renter.files["1"] = &file{ name: "one", } // Delete a different file. err = rt.renter.DeleteFile("one") if err != ErrUnknownPath { t.Error("Expected ErrUnknownPath, got", err) } // Delete the file. err = rt.renter.DeleteFile("1") if err != nil { t.Error(err) } if len(rt.renter.FileList()) != 0 { t.Error("file was deleted, but is still reported in FileList") } // Put a file in the renter, then rename it. f := newTestingFile() f.name = "1" rt.renter.files[f.name] = f rt.renter.RenameFile(f.name, "one") // Call delete on the previous name. err = rt.renter.DeleteFile("1") if err != ErrUnknownPath { t.Error("Expected ErrUnknownPath, got", err) } // Call delete on the new name. err = rt.renter.DeleteFile("one") if err != nil { t.Error(err) } // Check that all .sia files have been deleted. var walkStr string filepath.Walk(rt.renter.persistDir, func(path string, _ os.FileInfo, _ error) error { // capture only .sia files if filepath.Ext(path) == ".sia" { rel, _ := filepath.Rel(rt.renter.persistDir, path) // strip testdir prefix walkStr += rel } return nil }) expWalkStr := "" if walkStr != expWalkStr { t.Fatalf("Bad walk string: expected %q, got %q", expWalkStr, walkStr) } } // TestRenterFileList probes the FileList method of the renter type. func TestRenterFileList(t *testing.T) { if testing.Short() { t.SkipNow() } rt, err := newRenterTester("TestRenterFileList") if err != nil { t.Fatal(err) } defer rt.Close() // Get the file list of an empty renter. if len(rt.renter.FileList()) != 0 { t.Error("FileList has non-zero length for empty renter?") } // Put a file in the renter. rsc, _ := NewRSCode(1, 1) rt.renter.files["1"] = &file{ name: "one", erasureCode: rsc, pieceSize: 1, } if len(rt.renter.FileList()) != 1 { t.Error("FileList is not returning the only file in the renter") } if rt.renter.FileList()[0].SiaPath != "one" { t.Error("FileList is not returning the correct filename for the only file") } // Put multiple files in the renter. rt.renter.files["2"] = &file{ name: "two", erasureCode: rsc, pieceSize: 1, } if len(rt.renter.FileList()) != 2 { t.Error("FileList is not returning both files in the renter") } files := rt.renter.FileList() if !((files[0].SiaPath == "one" || files[0].SiaPath == "two") && (files[1].SiaPath == "one" || files[1].SiaPath == "two") && (files[0].SiaPath != files[1].SiaPath)) { t.Error("FileList is returning wrong names for the files:", files[0].SiaPath, files[1].SiaPath) } } // TestRenterRenameFile probes the rename method of the renter. func TestRenterRenameFile(t *testing.T) { rt, err := newRenterTester("TestRenterRenameFile") if err != nil { t.Fatal(err) } defer rt.Close() // Rename a file that doesn't exist. err = rt.renter.RenameFile("1", "1a") if err != ErrUnknownPath { t.Error("Expecting ErrUnknownPath:", err) } // Rename a file that does exist. f := newTestingFile() f.name = "1" rt.renter.files["1"] = f err = rt.renter.RenameFile("1", "1a") if err != nil { t.Fatal(err) } files := rt.renter.FileList() if len(files) != 1 { t.Fatal("FileList has unexpected number of files:", len(files)) } if files[0].SiaPath != "1a" { t.Errorf("RenameFile failed: expected 1a, got %v", files[0].SiaPath) } // Rename a file to an existing name. f2 := newTestingFile() f2.name = "1" rt.renter.files["1"] = f2 err = rt.renter.RenameFile("1", "1a") if err != ErrPathOverload { t.Error("Expecting ErrPathOverload, got", err) } // Rename a file to the same name. err = rt.renter.RenameFile("1", "1") if err != ErrPathOverload { t.Error("Expecting ErrPathOverload, got", err) } } Sia-1.0.4/modules/renter/hostdb/000077500000000000000000000000001302033070500164535ustar00rootroot00000000000000Sia-1.0.4/modules/renter/hostdb/dependencies.go000066400000000000000000000034531302033070500214350ustar00rootroot00000000000000package hostdb import ( "net" "path/filepath" "time" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" ) // These interfaces define the HostDB's dependencies. Using the smallest // interface possible makes it easier to mock these dependencies in testing. type ( consensusSet interface { ConsensusSetSubscribe(modules.ConsensusSetSubscriber, modules.ConsensusChangeID) error } dialer interface { DialTimeout(modules.NetAddress, time.Duration) (net.Conn, error) } sleeper interface { Sleep(time.Duration) } persister interface { save(hdbPersist) error saveSync(hdbPersist) error load(*hdbPersist) error } ) // stdDialer implements the dialer interface via net.DialTimeout. type stdDialer struct{} func (d stdDialer) DialTimeout(addr modules.NetAddress, timeout time.Duration) (net.Conn, error) { return net.DialTimeout("tcp", string(addr), timeout) } // stdSleeper implements the sleeper interface via time.Sleep. type stdSleeper struct{} func (s stdSleeper) Sleep(d time.Duration) { time.Sleep(d) } // stdPersist implements the persister interface via persist.SaveFile and // persist.LoadFile. The metadata and filename required by these functions is // internal to stdPersist. type stdPersist struct { meta persist.Metadata filename string } func (p *stdPersist) save(data hdbPersist) error { return persist.SaveFile(p.meta, data, p.filename) } func (p *stdPersist) saveSync(data hdbPersist) error { return persist.SaveFileSync(p.meta, data, p.filename) } func (p *stdPersist) load(data *hdbPersist) error { return persist.LoadFile(p.meta, data, p.filename) } func newPersist(dir string) *stdPersist { return &stdPersist{ meta: persist.Metadata{ Header: "HostDB Persistence", Version: "0.5", }, filename: filepath.Join(dir, "hostdb.json"), } } Sia-1.0.4/modules/renter/hostdb/hostdb.go000066400000000000000000000110601302033070500202630ustar00rootroot00000000000000// Package hostdb provides a HostDB object that implements the renter.hostDB // interface. The blockchain is scanned for host announcements and hosts that // are found get added to the host database. The database continually scans the // set of hosts it has found and updates who is online. package hostdb // TODO: There should be some mechanism that detects if the number of active // hosts is low. Then either the user can be informed, or the hostdb can start // scanning hosts that have been offline for a while and are no longer // prioritized by the scan loop. // TODO: There should be some mechanism for detecting if the hostdb cannot // connect to the internet. If it cannot, hosts should not be penalized for // appearing to be offline, because they may not actually be offline and it'll // unfairly over-penalize the hosts with the highest uptime. import ( "errors" "os" "path/filepath" "sync" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" siasync "github.com/NebulousLabs/Sia/sync" "github.com/NebulousLabs/Sia/types" ) const ( // scanPoolSize sets the buffer size of the channel that holds hosts which // need to be scanned. A thread pool pulls from the scan pool to query // hosts that are due for an update. scanPoolSize = 1000 ) var ( errNilCS = errors.New("cannot create hostdb with nil consensus set") ) // The HostDB is a database of potential hosts. It assigns a weight to each // host based on their hosting parameters, and then can select hosts at random // for uploading files. type HostDB struct { // dependencies dialer dialer log *persist.Logger mu sync.RWMutex persist persister sleeper sleeper tg siasync.ThreadGroup // The hostTree is the root node of the tree that organizes hosts by // weight. The tree is necessary for selecting weighted hosts at // random. 'activeHosts' provides a lookup from hostname to the the // corresponding node, as the hostTree is unsorted. A host is active if // it is currently responding to queries about price and other // settings. hostTree *hostNode activeHosts map[modules.NetAddress]*hostNode // allHosts is a simple list of all known hosts by their network address, // including hosts that are currently offline. allHosts map[modules.NetAddress]*hostEntry // the scanPool is a set of hosts that need to be scanned. There are a // handful of goroutines constantly waiting on the channel for hosts to // scan. scanList []*hostEntry scanPool chan *hostEntry scanWait bool blockHeight types.BlockHeight lastChange modules.ConsensusChangeID } // New returns a new HostDB. func New(cs consensusSet, persistDir string) (*HostDB, error) { // Check for nil inputs. if cs == nil { return nil, errNilCS } // Create the persist directory if it does not yet exist. err := os.MkdirAll(persistDir, 0700) if err != nil { return nil, err } // Create the logger. logger, err := persist.NewFileLogger(filepath.Join(persistDir, "hostdb.log")) if err != nil { return nil, err } // Create HostDB using production dependencies. return newHostDB(cs, stdDialer{}, stdSleeper{}, newPersist(persistDir), logger) } // newHostDB creates a HostDB using the provided dependencies. It loads the old // persistence data, spawns the HostDB's scanning threads, and subscribes it to // the consensusSet. func newHostDB(cs consensusSet, d dialer, s sleeper, p persister, l *persist.Logger) (*HostDB, error) { // Create the HostDB object. hdb := &HostDB{ dialer: d, sleeper: s, persist: p, log: l, // TODO: should index by pubkey, not ip activeHosts: make(map[modules.NetAddress]*hostNode), allHosts: make(map[modules.NetAddress]*hostEntry), scanPool: make(chan *hostEntry, scanPoolSize), } // Load the prior persistence structures. err := hdb.load() if err != nil && !os.IsNotExist(err) { return nil, err } err = cs.ConsensusSetSubscribe(hdb, hdb.lastChange) if err == modules.ErrInvalidConsensusChangeID { hdb.lastChange = modules.ConsensusChangeBeginning // clear the host sets hdb.activeHosts = make(map[modules.NetAddress]*hostNode) hdb.allHosts = make(map[modules.NetAddress]*hostEntry) // subscribe again using the new ID err = cs.ConsensusSetSubscribe(hdb, hdb.lastChange) } if err != nil { return nil, errors.New("hostdb subscription failed: " + err.Error()) } // Spin up the host scanning processes. for i := 0; i < scanningThreads; i++ { go hdb.threadedProbeHosts() } go hdb.threadedScan() return hdb, nil } // Close closes the hostdb, terminating its scanning threads func (hdb *HostDB) Close() error { return hdb.tg.Stop() } Sia-1.0.4/modules/renter/hostdb/hostdb_test.go000066400000000000000000000026121302033070500213250ustar00rootroot00000000000000package hostdb import ( "io/ioutil" "os" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" ) // bareHostDB returns a HostDB with its fields initialized, but without any // dependencies or scanning threads. It is only intended for use in unit tests. func bareHostDB() *HostDB { return &HostDB{ log: persist.NewLogger(ioutil.Discard), activeHosts: make(map[modules.NetAddress]*hostNode), allHosts: make(map[modules.NetAddress]*hostEntry), scanPool: make(chan *hostEntry, scanPoolSize), } } // newStub is used to test the New function. It implements all of the hostdb's // dependencies. type newStub struct{} // consensus set stubs func (newStub) ConsensusSetSubscribe(modules.ConsensusSetSubscriber, modules.ConsensusChangeID) error { return nil } // TestNew tests the New function. func TestNew(t *testing.T) { // Using a stub implementation of the dependencies is fine, as long as its // non-nil. var stub newStub dir := build.TempDir("hostdb", "TestNew") // Sane values. _, err := New(stub, dir) if err != nil { t.Fatalf("expected nil, got %v", err) } // Nil consensus set. _, err = New(nil, dir) if err != errNilCS { t.Fatalf("expected %v, got %v", errNilCS, err) } // Bad persistDir. _, err = New(stub, "") if !os.IsNotExist(err) { t.Fatalf("expected invalid directory, got %v", err) } } Sia-1.0.4/modules/renter/hostdb/hostentry.go000066400000000000000000000070761302033070500210530ustar00rootroot00000000000000package hostdb import ( "bytes" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // A hostEntry represents a host on the network. type hostEntry struct { modules.HostDBEntry FirstSeen types.BlockHeight Weight types.Currency Reliability types.Currency Online bool } // insertHost adds a host entry to the state. The host will be inserted into // the set of all hosts, and if it is online and responding to requests it will // be put into the list of active hosts. // // TODO: Function should return an error. func (hdb *HostDB) insertHost(host modules.HostDBEntry) { // Remove garbage hosts and local hosts (but allow local hosts in testing). if err := host.NetAddress.IsValid(); err != nil { hdb.log.Debugf("WARN: host '%v' has an invalid NetAddress: %v", host.NetAddress, err) return } // Don't do anything if we've already seen this host and the public key is // the same. if knownHost, exists := hdb.allHosts[host.NetAddress]; exists && bytes.Equal(host.PublicKey.Key, knownHost.PublicKey.Key) { return } // Create hostEntry and add to allHosts. h := &hostEntry{ FirstSeen: hdb.blockHeight, HostDBEntry: host, Reliability: DefaultReliability, } hdb.allHosts[host.NetAddress] = h // Add the host to the scan queue. If the scan is successful, the host // will be placed in activeHosts. hdb.queueHostEntry(h) } // Remove deletes an entry from the hostdb. func (hdb *HostDB) removeHost(addr modules.NetAddress) error { // See if the node is in the set of active hosts. node, exists := hdb.activeHosts[addr] if exists { node.removeNode() delete(hdb.activeHosts, addr) } // Remove the node from all hosts. delete(hdb.allHosts, addr) return nil } // Host returns the HostSettings associated with the specified NetAddress. If // no matching host is found, Host returns false. func (hdb *HostDB) Host(addr modules.NetAddress) (modules.HostDBEntry, bool) { hdb.mu.Lock() defer hdb.mu.Unlock() entry, ok := hdb.allHosts[addr] if !ok || entry == nil { return modules.HostDBEntry{}, false } return entry.HostDBEntry, true } // ActiveHosts returns the hosts that can be randomly selected out of the // hostdb, sorted by preference. func (hdb *HostDB) ActiveHosts() (activeHosts []modules.HostDBEntry) { hdb.mu.RLock() numHosts := len(hdb.activeHosts) hdb.mu.RUnlock() // Get the hosts using RandomHosts so that they are in sorted order. sortedHosts := hdb.RandomHosts(numHosts, nil) return sortedHosts } // AllHosts returns all of the hosts known to the hostdb, including the // inactive ones. func (hdb *HostDB) AllHosts() (allHosts []modules.HostDBEntry) { hdb.mu.RLock() defer hdb.mu.RUnlock() for _, entry := range hdb.allHosts { allHosts = append(allHosts, entry.HostDBEntry) } return } // AverageContractPrice returns the average price of a host. func (hdb *HostDB) AverageContractPrice() types.Currency { // maybe a more sophisticated way of doing this var totalPrice types.Currency sampleSize := 18 hosts := hdb.RandomHosts(sampleSize, nil) if len(hosts) == 0 { return totalPrice } for _, host := range hosts { totalPrice = totalPrice.Add(host.ContractPrice) } return totalPrice.Div64(uint64(len(hosts))) } // IsOffline reports whether a host is offline. If the HostDB has no record of // the host, IsOffline will return false. // // TODO: Is this behavior that makes sense? func (hdb *HostDB) IsOffline(addr modules.NetAddress) bool { hdb.mu.RLock() defer hdb.mu.RUnlock() if _, ok := hdb.activeHosts[addr]; ok { return false } if h, ok := hdb.allHosts[addr]; ok { return !h.Online } return false } Sia-1.0.4/modules/renter/hostdb/hostentry_test.go000066400000000000000000000076531302033070500221130ustar00rootroot00000000000000package hostdb import ( "testing" "time" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // TestInsertHost tests the insertHost method, which also depends on the // scanHostEntry method. func TestInsertHost(t *testing.T) { hdb := bareHostDB() // invalid host should not be scanned var dbe modules.HostDBEntry dbe.NetAddress = "foo" hdb.insertHost(dbe) select { case <-hdb.scanPool: t.Error("invalid host was added to scan pool") case <-time.After(100 * time.Millisecond): } // valid host should be scanned dbe.NetAddress = "foo.com:1234" hdb.insertHost(dbe) select { case <-hdb.scanPool: case <-time.After(time.Second): t.Error("host was not scanned") } // duplicate host should not be scanned hdb.insertHost(dbe) select { case <-hdb.scanPool: t.Error("duplicate host was added to scan pool") case <-time.After(100 * time.Millisecond): } } // TestActiveHosts tests the ActiveHosts method. func TestActiveHosts(t *testing.T) { hdb := bareHostDB() // empty if hosts := hdb.ActiveHosts(); len(hosts) != 0 { t.Errorf("wrong number of hosts: expected %v, got %v", 0, len(hosts)) } // with one host h1 := new(hostEntry) h1.NetAddress = "foo" h1.Weight = types.NewCurrency64(1) h1.AcceptingContracts = true hdb.insertNode(h1) if hosts := hdb.ActiveHosts(); len(hosts) != 1 { t.Errorf("wrong number of hosts: expected %v, got %v", 1, len(hosts)) } else if hosts[0].NetAddress != h1.NetAddress { t.Errorf("ActiveHosts returned wrong host: expected %v, got %v", h1.NetAddress, hosts[0].NetAddress) } // with multiple hosts h2 := new(hostEntry) h2.NetAddress = "bar" h2.Weight = types.NewCurrency64(1) h2.AcceptingContracts = true hdb.insertNode(h2) if hosts := hdb.ActiveHosts(); len(hosts) != 2 { t.Errorf("wrong number of hosts: expected %v, got %v", 2, len(hosts)) } else if hosts[0].NetAddress != h1.NetAddress && hosts[1].NetAddress != h1.NetAddress { t.Errorf("ActiveHosts did not contain an inserted host: %v (missing %v)", hosts, h1.NetAddress) } else if hosts[0].NetAddress != h2.NetAddress && hosts[1].NetAddress != h2.NetAddress { t.Errorf("ActiveHosts did not contain an inserted host: %v (missing %v)", hosts, h2.NetAddress) } } // TestAverageContractPrice tests the AverageContractPrice method, which also depends on the // randomHosts method. func TestAverageContractPrice(t *testing.T) { hdb := bareHostDB() // empty if avg := hdb.AverageContractPrice(); !avg.IsZero() { t.Error("average of empty hostdb should be zero:", avg) } // with one host h1 := new(hostEntry) h1.NetAddress = "foo" h1.ContractPrice = types.NewCurrency64(100) h1.Weight = baseWeight h1.AcceptingContracts = true hdb.insertNode(h1) if avg := hdb.AverageContractPrice(); avg.Cmp(h1.ContractPrice) != 0 { t.Error("average of one host should be that host's price:", avg) } // with two hosts h2 := new(hostEntry) h2.NetAddress = "bar" h2.ContractPrice = types.NewCurrency64(300) h2.Weight = baseWeight h2.AcceptingContracts = true hdb.insertNode(h2) if len(hdb.activeHosts) != 2 { t.Error("host was not added:", hdb.activeHosts) } if avg := hdb.AverageContractPrice(); avg.Cmp64(200) != 0 { t.Error("average of two hosts should be their sum/2:", avg) } } // TestIsOffline tests the IsOffline method. func TestIsOffline(t *testing.T) { hdb := &HostDB{ allHosts: map[modules.NetAddress]*hostEntry{ "foo.com:1234": {Online: true}, "bar.com:1234": {Online: false}, "baz.com:1234": {Online: true}, }, activeHosts: map[modules.NetAddress]*hostNode{ "foo.com:1234": nil, }, scanPool: make(chan *hostEntry), } tests := []struct { addr modules.NetAddress offline bool }{ {"foo.com:1234", false}, {"bar.com:1234", true}, {"baz.com:1234", false}, {"quux.com:1234", false}, } for _, test := range tests { if offline := hdb.IsOffline(test.addr); offline != test.offline { t.Errorf("IsOffline(%v) = %v, expected %v", test.addr, offline, test.offline) } } } Sia-1.0.4/modules/renter/hostdb/hostweight.go000066400000000000000000000133271302033070500211750ustar00rootroot00000000000000package hostdb import ( "math/big" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/types" ) var ( // Because most weights would otherwise be fractional, we set the base // weight to 10^150 to give ourselves lots of precision when determing the // weight of a host baseWeight = types.NewCurrency(new(big.Int).Exp(big.NewInt(10), big.NewInt(150), nil)) // requiredStorage indicates the amount of storage that the host must be // offering in order to be considered a valuable/worthwhile host. requiredStorage = func() uint64 { switch build.Release { case "dev": return 1e6 case "standard": return 1e9 case "testing": return 1e3 default: panic("incorrect/missing value for requiredStorage constant") } }() ) // calculateHostWeight returns the weight of a host according to the settings of // the host database entry. Currently, only the price is considered. func calculateHostWeight(currentHeight types.BlockHeight, entry hostEntry) (weight types.Currency) { // Prices tiered as follows: // - the storage price is presented as 'per block per byte' // - the contract price is presented as a flat rate // - the upload bandwidth price is per byte // - the download bandwidth price is per byte // // The hostdb will naively assume the following for now: // - each contract covers 6 weeks of storage (default is 12 weeks, but // renewals occur at midpoint) - 6048 blocks - and 10GB of storage. // - uploads happen once per 12 weeks (average lifetime of a file is 12 weeks) // - downloads happen once per 6 weeks (files are on average downloaded twice throughout lifetime) // // In the future, the renter should be able to track average user behavior // and adjust accordingly. This flexibility will be added later. adjustedContractPrice := entry.ContractPrice.Div64(6048).Div64(10e9) // Adjust contract price to match 10GB for 6 weeks. adjustedUploadPrice := entry.UploadBandwidthPrice.Div64(24192) // Adjust upload price to match a single upload over 24 weeks. adjustedDownloadPrice := entry.DownloadBandwidthPrice.Div64(12096) // Adjust download price to match one download over 12 weeks. siafundFee := adjustedContractPrice.Add(adjustedUploadPrice).Add(adjustedDownloadPrice).Add(entry.Collateral).MulTax() totalPrice := entry.StoragePrice.Add(adjustedContractPrice).Add(adjustedUploadPrice).Add(adjustedDownloadPrice).Add(siafundFee) // Set the weight to the base weight, and then divide it by the price // raised to the fifth power. This means that a host which has half the // total price will be 32x as likely to be selected. A host with a quarter // the total price will be 1024x as likely to be selected, and so on. weight = baseWeight if !totalPrice.IsZero() { // To avoid a divide-by-zero error, this operation is only performed on // non-zero prices. weight = baseWeight.Div(totalPrice).Div(totalPrice).Div(totalPrice).Div(totalPrice).Div(totalPrice) } // Enact penalties if the host does not have very much storage remaining. if entry.RemainingStorage < 200*requiredStorage { weight = weight.Div64(2) // 2x total penalty } if entry.RemainingStorage < 100*requiredStorage { weight = weight.Div64(3) // 6x total penalty } if entry.RemainingStorage < 50*requiredStorage { weight = weight.Div64(4) // 24x total penalty } if entry.RemainingStorage < 25*requiredStorage { weight = weight.Div64(5) // 95x total penalty } if entry.RemainingStorage < 10*requiredStorage { weight = weight.Div64(6) // 570x total penalty } if entry.RemainingStorage < 5*requiredStorage { weight = weight.Div64(10) // 5,700x total penalty } if entry.RemainingStorage < requiredStorage { weight = weight.Div64(100) // 570,000 total penalty } // Enact penalities for hosts running older versions. if build.VersionCmp(entry.Version, "1.0.3") < 0 { weight = weight.Div64(5) // 5x total penalty. } if build.VersionCmp(entry.Version, "1.0.0") < 0 { weight = weight.Div64(10) // 50x total penalty. } // Enact penalities for newer hosts, as it's less certain that they will // have reliable uptime. if currentHeight >= entry.FirstSeen { age := currentHeight - entry.FirstSeen if age < 6000 { weight = weight.Div64(2) // 2x total } if age < 4000 { weight = weight.Div64(2) // 4x total } if age < 2000 { weight = weight.Div64(4) // 16x total } if age < 1000 { weight = weight.Div64(4) // 64x total } if age < 288 { weight = weight.Div64(10) // 640x total } } else { // Shouldn't happen, but the usecase is covered anyway. weight = weight.Div64(1000) // Because something weird is happening, don't trust this host very much. } // Account for collateral. Collateral has a somewhat complicated // relationship with price, because raising the collateral inherently // raises the price for renters. If the host's score increases linearly to // the amount of collateral they put up, then the host will gain score from // increasing collateral until the siafund fee makes up about 15% of the // total price. After that, the host will actually lose points for having // too much collateral. // // The renter has control over how much collateral the host uses. // Currently, this control is not implemented, so the hosts are penalized // for setting very high collateral values. Once the renter is clamping the // amount being spent on collateral, the hostdb can also clamp the amount // of collateral being taken into account by the host, to optimize the // host's score for the renter's needs. if entry.Collateral.IsZero() { // Instead of zeroing out the weight, just return the weight as though // the collateral is 1 hasting. Competitively speaking, this is // effectively zero. return weight } weight = weight.Mul(entry.Collateral) return weight } Sia-1.0.4/modules/renter/hostdb/hostweight_test.go000066400000000000000000000027471302033070500222400ustar00rootroot00000000000000package hostdb import ( "testing" "github.com/NebulousLabs/Sia/types" ) func calculateWeightFromUInt64Price(price uint64) (weight types.Currency) { var entry hostEntry entry.RemainingStorage = 250e3 entry.StoragePrice = types.NewCurrency64(price) return calculateHostWeight(0, entry) } func TestHostWeightDistinctPrices(t *testing.T) { // Create two identical entries, except that one has a price that is 2x the // other. The weight returned by hostWeight should be 1/8 for the more // expensive host. weight1 := calculateWeightFromUInt64Price(3) weight2 := calculateWeightFromUInt64Price(6) expectedWeight := weight1.Div64(32) if weight2.Cmp(expectedWeight) != 0 { t.Error("Weight of expensive host is not the correct value.") } } func TestHostWeightIdenticalPrices(t *testing.T) { weight1 := calculateWeightFromUInt64Price(42) weight2 := calculateWeightFromUInt64Price(42) if weight1.Cmp(weight2) != 0 { t.Error("Weight of identically priced hosts should be equal.") } } func TestHostWeightWithOnePricedZero(t *testing.T) { weight1 := calculateWeightFromUInt64Price(5) weight2 := calculateWeightFromUInt64Price(0) if weight1.Cmp(weight2) >= 0 { t.Error("Zero-priced host should have higher weight than nonzero-priced host.") } } func TestHostWeightWithBothPricesZero(t *testing.T) { weight1 := calculateWeightFromUInt64Price(0) weight2 := calculateWeightFromUInt64Price(0) if weight1.Cmp(weight2) != 0 { t.Error("Weight of two zero-priced hosts should be equal.") } } Sia-1.0.4/modules/renter/hostdb/persist.go000066400000000000000000000025131302033070500204740ustar00rootroot00000000000000package hostdb import ( "github.com/NebulousLabs/Sia/modules" ) // hdbPersist defines what HostDB data persists across sessions. type hdbPersist struct { AllHosts []hostEntry ActiveHosts []hostEntry LastChange modules.ConsensusChangeID } // persistData returns the data in the hostdb that will be saved to disk. func (hdb *HostDB) persistData() hdbPersist { var data hdbPersist for _, entry := range hdb.allHosts { data.AllHosts = append(data.AllHosts, *entry) } for _, node := range hdb.activeHosts { data.ActiveHosts = append(data.ActiveHosts, *node.hostEntry) } data.LastChange = hdb.lastChange return data } // save saves the hostdb persistence data to disk. func (hdb *HostDB) save() error { return hdb.persist.save(hdb.persistData()) } // saveSync saves the hostdb persistence data to disk and then syncs to disk. func (hdb *HostDB) saveSync() error { return hdb.persist.saveSync(hdb.persistData()) } // load loads the hostdb persistence data from disk. func (hdb *HostDB) load() error { var data hdbPersist err := hdb.persist.load(&data) if err != nil { return err } for i := range data.AllHosts { hdb.allHosts[data.AllHosts[i].NetAddress] = &data.AllHosts[i] } for i := range data.ActiveHosts { hdb.insertNode(hdb.allHosts[data.ActiveHosts[i].NetAddress]) } hdb.lastChange = data.LastChange return nil } Sia-1.0.4/modules/renter/hostdb/persist_test.go000066400000000000000000000106621302033070500215370ustar00rootroot00000000000000package hostdb import ( "crypto/rand" "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // memPersist implements the persister interface in-memory. type memPersist hdbPersist func (m *memPersist) save(data hdbPersist) error { *m = memPersist(data); return nil } func (m *memPersist) saveSync(data hdbPersist) error { *m = memPersist(data); return nil } func (m memPersist) load(data *hdbPersist) error { *data = hdbPersist(m); return nil } // TestSaveLoad tests that the hostdb can save and load itself. func TestSaveLoad(t *testing.T) { // create hostdb with mocked persist dependency hdb := bareHostDB() hdb.persist = new(memPersist) // add some fake hosts var host1, host2, host3 hostEntry host1.NetAddress = "foo" host2.NetAddress = "bar" host3.NetAddress = "baz" hdb.allHosts = map[modules.NetAddress]*hostEntry{ host1.NetAddress: &host1, host2.NetAddress: &host2, host3.NetAddress: &host3, } hdb.activeHosts = map[modules.NetAddress]*hostNode{ host1.NetAddress: {hostEntry: &host1}, host2.NetAddress: {hostEntry: &host2}, host3.NetAddress: {hostEntry: &host3}, } hdb.lastChange = modules.ConsensusChangeID{1, 2, 3} // save and reload err := hdb.save() if err != nil { t.Fatal(err) } err = hdb.load() if err != nil { t.Fatal(err) } // check that LastChange was loaded if hdb.lastChange != (modules.ConsensusChangeID{1, 2, 3}) { t.Error("wrong consensus change ID was loaded:", hdb.lastChange) } // check that AllHosts was loaded _, ok0 := hdb.allHosts[host1.NetAddress] _, ok1 := hdb.allHosts[host2.NetAddress] _, ok2 := hdb.allHosts[host3.NetAddress] if !ok0 || !ok1 || !ok2 || len(hdb.allHosts) != 3 { t.Fatal("allHosts was not restored properly:", hdb.allHosts) } // check that ActiveHosts was loaded _, ok0 = hdb.activeHosts[host1.NetAddress] _, ok1 = hdb.activeHosts[host2.NetAddress] _, ok2 = hdb.activeHosts[host3.NetAddress] if !ok0 || !ok1 || !ok2 || len(hdb.activeHosts) != 3 { t.Fatal("active was not restored properly:", hdb.activeHosts) } } // rescanCS is a barebones implementation of a consensus set that can be // subscribed to. type rescanCS struct { changes []modules.ConsensusChange } func (cs *rescanCS) addBlock(b types.Block) { cc := modules.ConsensusChange{ AppliedBlocks: []types.Block{b}, } rand.Read(cc.ID[:]) cs.changes = append(cs.changes, cc) } func (cs *rescanCS) ConsensusSetSubscribe(s modules.ConsensusSetSubscriber, lastChange modules.ConsensusChangeID) error { var start int if lastChange != (modules.ConsensusChangeID{}) { start = -1 for i, cc := range cs.changes { if cc.ID == lastChange { start = i break } } if start == -1 { return modules.ErrInvalidConsensusChangeID } } for _, cc := range cs.changes[start:] { s.ProcessConsensusChange(cc) } return nil } // TestRescan tests that the hostdb will rescan the blockchain properly. func TestRescan(t *testing.T) { // create hostdb with mocked persist dependency hdb := bareHostDB() hdb.persist = new(memPersist) // add some fake hosts var host1, host2, host3 hostEntry host1.NetAddress = "foo" host2.NetAddress = "bar" host3.NetAddress = "baz" hdb.allHosts = map[modules.NetAddress]*hostEntry{ host1.NetAddress: &host1, host2.NetAddress: &host2, host3.NetAddress: &host3, } hdb.activeHosts = map[modules.NetAddress]*hostNode{ host1.NetAddress: {hostEntry: &host1}, host2.NetAddress: {hostEntry: &host2}, host3.NetAddress: {hostEntry: &host3}, } // use a bogus change ID hdb.lastChange = modules.ConsensusChangeID{1, 2, 3} // save the hostdb err := hdb.save() if err != nil { t.Fatal(err) } // create a mocked consensus set with a different host announcement annBytes, err := makeSignedAnnouncement("quux.com:1234") if err != nil { t.Fatal(err) } announceBlock := types.Block{ Transactions: []types.Transaction{{ ArbitraryData: [][]byte{annBytes}, }}, } cs := new(rescanCS) cs.addBlock(announceBlock) // Reload the hostdb using the same persist and the mocked consensus set. // The old change ID will be rejected, causing a rescan, which should // discover the new announcement. hdb, err = newHostDB(cs, stdDialer{}, stdSleeper{}, hdb.persist, hdb.log) if err != nil { t.Fatal(err) } if len(hdb.allHosts) != 1 { t.Fatal("hostdb rescan resulted in wrong host set:", hdb.allHosts) } if _, exists := hdb.allHosts["quux.com:1234"]; !exists { t.Fatal("hostdb rescan resulted in wrong host set:", hdb.allHosts) } } Sia-1.0.4/modules/renter/hostdb/scan.go000066400000000000000000000223711302033070500177330ustar00rootroot00000000000000package hostdb // scan.go contains the functions which periodically scan the list of all hosts // to see which hosts are online or offline, and to get any updates to the // settings of the hosts. import ( "bytes" "crypto/rand" "math/big" "net" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) const ( defaultScanSleep = 1*time.Hour + 37*time.Minute maxScanSleep = 4 * time.Hour minScanSleep = 1*time.Hour + 20*time.Minute maxActiveHosts = 500 inactiveHostCheckupQuantity = 1000 maxSettingsLen = 2e3 hostRequestTimeout = 60 * time.Second hostScanDeadline = 60 * time.Second // scanningThreads is the number of threads that will be probing hosts for // their settings and checking for reliability. scanningThreads = 50 ) // Reliability is a measure of a host's uptime. var ( MaxReliability = types.NewCurrency64(500) // Given the scanning defaults, about 6 weeks of survival. DefaultReliability = types.NewCurrency64(150) // Given the scanning defaults, about 2 week of survival. UnreachablePenalty = types.NewCurrency64(1) ) // queueHostEntry will add a host entry to the list of entries waiting to be // scanned. If there is no thread that is currently walking through the scan // list, one will be created and it will persist until shutdown or until the // scan list is empty. func (hdb *HostDB) queueHostEntry(entry *hostEntry) { // Add the entry to a waitlist, then check if any thread is currently // emptying the waitlist. If not, spawn a thread to empty the waitlist. hdb.scanList = append(hdb.scanList, entry) if hdb.scanWait { // Another thread is emptying the scan list, nothing to worry about. return } // Nobody is emptying the scan list, volunteer. if hdb.tg.Add() != nil { // Hostdb is shutting down, don't spin up another thread. return } hdb.scanWait = true go func() { defer hdb.tg.Done() for { hdb.mu.Lock() if len(hdb.scanList) == 0 { // Scan list is empty, can exit. Let the world know that nobody // is emptying the scan list anymore. hdb.scanWait = false hdb.mu.Unlock() return } // Get the next host, shrink the scan list. entry := hdb.scanList[0] hdb.scanList = hdb.scanList[1:] hdb.mu.Unlock() // Block while we wait for an opening in the scan pool. select { case hdb.scanPool <- entry: // iterate again case <-hdb.tg.StopChan(): // quit return } } }() } // decrementReliability reduces the reliability of a node, moving it out of the // set of active hosts or deleting it entirely if necessary. func (hdb *HostDB) decrementReliability(addr modules.NetAddress, penalty types.Currency) { hdb.log.Debugln("reliability decrement issued for", addr) // Look up the entry and decrement the reliability. entry, exists := hdb.allHosts[addr] if !exists { // TODO: should panic here return } entry.Reliability = entry.Reliability.Sub(penalty) entry.Online = false // If the entry is in the active database, remove it from the active // database. node, exists := hdb.activeHosts[addr] if exists { hdb.log.Debugln("host is being pulled from list of active hosts", addr) node.removeNode() delete(hdb.activeHosts, entry.NetAddress) } // If the reliability has fallen to 0, remove the host from the // database entirely. if entry.Reliability.IsZero() { hdb.log.Debugln("host is being dropped from hostdb", addr) delete(hdb.allHosts, addr) } } // managedUpdateEntry updates an entry in the hostdb after a scan has taken // place. func (hdb *HostDB) managedUpdateEntry(entry *hostEntry, newSettings modules.HostExternalSettings, netErr error) { hdb.mu.Lock() defer hdb.mu.Unlock() // Regardless of whether the host responded, add it to allHosts. priorHost, exists := hdb.allHosts[entry.NetAddress] if !exists { hdb.allHosts[entry.NetAddress] = entry } // If the scan was unsuccessful, decrement the host's reliability. if netErr != nil { if exists && bytes.Equal(priorHost.PublicKey.Key, entry.PublicKey.Key) { // Only decrement the reliability if the public key in the // hostdb matches the public key in the host announcement - // the failure may just be a failed signature, indicating // the wrong public key. hdb.decrementReliability(entry.NetAddress, UnreachablePenalty) } return } // The host entry should be updated to reflect the new weight. The safety // properties of the tree require that the weight does not change while the // node is in the tree, so the node must be removed before the settings and // weight are changed. existingNode, exists := hdb.activeHosts[entry.NetAddress] if exists { existingNode.removeNode() delete(hdb.activeHosts, entry.NetAddress) } else if len(hdb.activeHosts) > maxActiveHosts { // We already have the maximum number of active hosts, do not add more. return } // Update the host settings, reliability, and weight. The old NetAddress // must be preserved. newSettings.NetAddress = entry.HostExternalSettings.NetAddress entry.HostExternalSettings = newSettings entry.Reliability = MaxReliability entry.Online = true entry.Weight = calculateHostWeight(hdb.blockHeight, *entry) hdb.insertNode(entry) // Sanity check - the node should be in the hostdb now. _, exists = hdb.activeHosts[entry.NetAddress] if !exists { hdb.log.Critical("Host was not added to the list of active hosts after the entry was updated.") } hdb.save() } // managedScanHost will connect to a host and grab the settings, verifying // uptime and updating to the host's preferences. func (hdb *HostDB) managedScanHost(hostEntry *hostEntry) { // Request settings from the queued host entry. // // A readlock is necessary when viewing the elements of the host entry. hdb.mu.RLock() netAddr := hostEntry.NetAddress pubKey := hostEntry.PublicKey hdb.mu.RUnlock() hdb.log.Debugln("Scanning", netAddr, pubKey) var settings modules.HostExternalSettings err := func() error { dialer := &net.Dialer{ Cancel: hdb.tg.StopChan(), Timeout: hostRequestTimeout, } conn, err := dialer.Dial("tcp", string(netAddr)) if err != nil { return err } connCloseChan := make(chan struct{}) go func() { select { case <-hdb.tg.StopChan(): case <-connCloseChan: } conn.Close() }() defer close(connCloseChan) conn.SetDeadline(time.Now().Add(hostScanDeadline)) err = encoding.WriteObject(conn, modules.RPCSettings) if err != nil { return err } var pubkey crypto.PublicKey copy(pubkey[:], pubKey.Key) return crypto.ReadSignedObject(conn, &settings, maxSettingsLen, pubkey) }() if err != nil { hdb.log.Debugln("Scanning", netAddr, pubKey, "failed:", err) } else { hdb.log.Debugln("Scanning", netAddr, pubKey, "succeeded") } // Update the host tree to have a new entry. hdb.managedUpdateEntry(hostEntry, settings, err) } // threadedProbeHosts tries to fetch the settings of a host. If successful, the // host is put in the set of active hosts. If unsuccessful, the host id deleted // from the set of active hosts. func (hdb *HostDB) threadedProbeHosts() { err := hdb.tg.Add() if err != nil { return } defer hdb.tg.Done() for { select { case <-hdb.tg.StopChan(): return case hostEntry := <-hdb.scanPool: hdb.managedScanHost(hostEntry) } } } // threadedScan is an ongoing function which will query the full set of hosts // every few hours to see who is online and available for uploading. func (hdb *HostDB) threadedScan() { err := hdb.tg.Add() if err != nil { return } defer hdb.tg.Done() for { // Determine who to scan. At most 'maxActiveHosts' will be scanned, // starting with the active hosts followed by a random selection of the // inactive hosts. func() { hdb.mu.Lock() defer hdb.mu.Unlock() // Scan all active hosts. for _, host := range hdb.activeHosts { hdb.queueHostEntry(host.hostEntry) } // Assemble all of the inactive hosts into a single array. var entries []*hostEntry for _, entry := range hdb.allHosts { _, exists := hdb.activeHosts[entry.NetAddress] if !exists { entries = append(entries, entry) } } // Generate a random ordering of up to inactiveHostCheckupQuantity // hosts. hostOrder, err := crypto.Perm(len(entries)) if err != nil { hdb.log.Println("ERR: could not generate random permutation:", err) } // Scan each host. for i := 0; i < len(hostOrder) && i < inactiveHostCheckupQuantity; i++ { hdb.queueHostEntry(entries[hostOrder[i]]) } }() // Sleep for a random amount of time before doing another round of // scanning. The minimums and maximums keep the scan time reasonable, // while the randomness prevents the scanning from always happening at // the same time of day or week. maxBig := big.NewInt(int64(maxScanSleep)) minBig := big.NewInt(int64(minScanSleep)) randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig)) if err != nil { build.Critical(err) // If there's an error, sleep for the default amount of time. defaultBig := big.NewInt(int64(defaultScanSleep)) randSleep = defaultBig.Sub(defaultBig, minBig) } // Sleep until it's time for the next scan cycle. select { case <-hdb.tg.StopChan(): return case <-time.After(time.Duration(randSleep.Int64()) + minScanSleep): } } } Sia-1.0.4/modules/renter/hostdb/scan_test.go000066400000000000000000000172631302033070500207760ustar00rootroot00000000000000package hostdb import ( "net" "testing" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // TestDecrementReliability tests the decrementReliability method. func TestDecrementReliability(t *testing.T) { hdb := bareHostDB() // Decrementing a non-existent host should be a no-op. // NOTE: can't check any post-conditions here; only indication of correct // behavior is that the test doesn't panic. hdb.decrementReliability("foo", types.NewCurrency64(0)) // Add a host to allHosts and activeHosts. Decrementing it should remove it // from activeHosts. h := new(hostEntry) h.NetAddress = "foo" h.Reliability = types.NewCurrency64(1) hdb.allHosts[h.NetAddress] = h hdb.activeHosts[h.NetAddress] = &hostNode{hostEntry: h} hdb.decrementReliability(h.NetAddress, types.NewCurrency64(0)) if len(hdb.ActiveHosts()) != 0 { t.Error("decrementing did not remove host from activeHosts") } // Decrement reliability to 0. This should remove the host from allHosts. hdb.decrementReliability(h.NetAddress, h.Reliability) if len(hdb.AllHosts()) != 0 { t.Error("decrementing did not remove host from allHosts") } } // probeDialer is used to test the threadedProbeHosts method. A simple type // alias is used so that it can easily be redefined during testing, allowing // multiple behaviors to be tested. type probeDialer func(modules.NetAddress, time.Duration) (net.Conn, error) func (dial probeDialer) DialTimeout(addr modules.NetAddress, timeout time.Duration) (net.Conn, error) { return dial(addr, timeout) } // TestThreadedProbeHosts tests the threadedProbeHosts method. func TestThreadedProbeHosts(t *testing.T) { t.Skip("incompatible concurrency patterns") if testing.Short() { t.SkipNow() } hdb := bareHostDB() hdb.persist = &memPersist{} // create a host to send to threadedProbeHosts sk, pk, err := crypto.GenerateKeyPair() if err != nil { t.Fatal(err) } h := new(hostEntry) h.NetAddress = "foo" h.AcceptingContracts = true h.PublicKey = types.SiaPublicKey{ Algorithm: types.SignatureEd25519, Key: pk[:], } h.Reliability = baseWeight // enough to withstand a few failures // define a helper function for running threadedProbeHosts. We send the // hostEntry, close the channel, and then call threadedProbeHosts. // threadedProbeHosts will receive the host, loop once, and return after // seeing the channel has closed. runProbe := func(h *hostEntry) { hdb.scanPool <- h close(hdb.scanPool) hdb.threadedProbeHosts() // reset hdb.scanPool hdb.scanPool = make(chan *hostEntry, 1) } // make the dial fail hdb.dialer = probeDialer(func(modules.NetAddress, time.Duration) (net.Conn, error) { return nil, net.UnknownNetworkError("fail") }) runProbe(h) if len(hdb.ActiveHosts()) != 0 { t.Error("unresponsive host was added") } // make the RPC fail hdb.dialer = probeDialer(func(modules.NetAddress, time.Duration) (net.Conn, error) { ourPipe, theirPipe := net.Pipe() ourPipe.Close() return theirPipe, nil }) runProbe(h) if len(hdb.ActiveHosts()) != 0 { t.Error("unresponsive host was added") } // normal host hdb.dialer = probeDialer(func(modules.NetAddress, time.Duration) (net.Conn, error) { // create an in-memory conn and spawn a goroutine to handle our half ourConn, theirConn := net.Pipe() go func() { // read the RPC encoding.ReadObject(ourConn, new(types.Specifier), types.SpecifierLen) // write host settings crypto.WriteSignedObject(ourConn, modules.HostExternalSettings{ AcceptingContracts: true, NetAddress: "probed", }, sk) ourConn.Close() }() return theirConn, nil }) runProbe(h) if len(hdb.ActiveHosts()) != 1 { t.Error("host was not added") } } // TestThreadedProbeHostsCorruption tests the threadedProbeHosts method, // specifically checking for corruption of the hostdb if the weight of a host // changes after a scan. func TestThreadedProbeHostsCorruption(t *testing.T) { t.Skip("incompatible concurrency patterns") if testing.Short() { t.SkipNow() } hdb := bareHostDB() hdb.persist = &memPersist{} // create a host to send to threadedProbeHosts sk, pk, err := crypto.GenerateKeyPair() if err != nil { t.Fatal(err) } h := new(hostEntry) h.NetAddress = "foo" h.AcceptingContracts = true h.PublicKey = types.SiaPublicKey{ Algorithm: types.SignatureEd25519, Key: pk[:], } h.Reliability = baseWeight // enough to withstand a few failures // define a helper function for running threadedProbeHosts. We send the // hostEntry, close the channel, and then call threadedProbeHosts. // threadedProbeHosts will receive the host, loop once, and return after // seeing the channel has closed. runProbe := func(h *hostEntry) { hdb.scanPool <- h close(hdb.scanPool) hdb.threadedProbeHosts() // reset hdb.scanPool hdb.scanPool = make(chan *hostEntry, 1) } // Add a normal host. hdb.dialer = probeDialer(func(modules.NetAddress, time.Duration) (net.Conn, error) { // create an in-memory conn and spawn a goroutine to handle our half ourConn, theirConn := net.Pipe() go func() { // read the RPC encoding.ReadObject(ourConn, new(types.Specifier), types.SpecifierLen) // write host settings crypto.WriteSignedObject(ourConn, modules.HostExternalSettings{ AcceptingContracts: true, StoragePrice: types.NewCurrency64(15e6), NetAddress: "probed", }, sk) ourConn.Close() }() return theirConn, nil }) runProbe(h) if len(hdb.ActiveHosts()) != 1 { t.Error("host was not added") } // Add the host again, this time changing the storage price, which will // change the weight of the host, which at one point would cause a // corruption of the host tree. hdb.dialer = probeDialer(func(modules.NetAddress, time.Duration) (net.Conn, error) { // create an in-memory conn and spawn a goroutine to handle our half ourConn, theirConn := net.Pipe() go func() { // read the RPC encoding.ReadObject(ourConn, new(types.Specifier), types.SpecifierLen) // write host settings crypto.WriteSignedObject(ourConn, modules.HostExternalSettings{ AcceptingContracts: true, StoragePrice: types.NewCurrency64(15e3), // Lower than the previous, to cause a higher weight. NetAddress: "probed", }, sk) ourConn.Close() }() return theirConn, nil }) runProbe(h) if len(hdb.ActiveHosts()) != 1 { t.Error("host was not added") } // Check that the host tree has not been corrupted. err = repeatCheck(hdb.hostTree) if err != nil { t.Error(err) } err = uniformTreeVerification(hdb, 1) if err != nil { t.Error(err) } } // TestThreadedScan tests the threadedScan method. func TestThreadedScan(t *testing.T) { hdb := bareHostDB() hdb.persist = &memPersist{} // use a real sleeper; this will prevent threadedScan from looping too // quickly. hdb.sleeper = stdSleeper{} // use a dummy dialer that always fails hdb.dialer = probeDialer(func(modules.NetAddress, time.Duration) (net.Conn, error) { return nil, net.UnknownNetworkError("fail") }) // create a host to be scanned h := new(hostEntry) h.NetAddress = "foo" h.Reliability = types.NewCurrency64(1) hdb.activeHosts[h.NetAddress] = &hostNode{hostEntry: h} // perform one scan go hdb.threadedScan() // host should be sent down scanPool select { case <-hdb.scanPool: case <-time.After(time.Second): t.Error("host was not scanned") } // remove the host from activeHosts and add it to allHosts hdb.mu.Lock() delete(hdb.activeHosts, h.NetAddress) hdb.allHosts[h.NetAddress] = h hdb.mu.Unlock() // perform one scan go hdb.threadedScan() // host should be sent down scanPool select { case <-hdb.scanPool: case <-time.After(time.Second): t.Error("host was not scanned") } } Sia-1.0.4/modules/renter/hostdb/update.go000066400000000000000000000051441302033070500202700ustar00rootroot00000000000000package hostdb import ( "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // findHostAnnouncements returns a list of the host announcements found within // a given block. No check is made to see that the ip address found in the // announcement is actually a valid ip address. func findHostAnnouncements(b types.Block) (announcements []modules.HostDBEntry) { for _, t := range b.Transactions { // the HostAnnouncement must be prefaced by the standard host // announcement string for _, arb := range t.ArbitraryData { addr, pubKey, err := modules.DecodeAnnouncement(arb) if err != nil { continue } // Add the announcement to the slice being returned. var host modules.HostDBEntry host.NetAddress = addr host.PublicKey = pubKey announcements = append(announcements, host) } } return } // ProcessConsensusChange will be called by the consensus set every time there // is a change in the blockchain. Updates will always be called in order. func (hdb *HostDB) ProcessConsensusChange(cc modules.ConsensusChange) { hdb.mu.Lock() defer hdb.mu.Unlock() // Update the hostdb's understanding of the block height. for _, block := range cc.RevertedBlocks { // Only doing the block check if the height is above zero saves hashing // and saves a nontrivial amount of time during IBD. if hdb.blockHeight > 0 || block.ID() != types.GenesisID { hdb.blockHeight-- } else if hdb.blockHeight != 0 { // Sanity check - if the current block is the genesis block, the // hostdb height should be set to zero. hdb.log.Critical("Hostdb has detected a genesis block, but the height of the hostdb is set to ", hdb.blockHeight) hdb.blockHeight = 0 } } for _, block := range cc.AppliedBlocks { // Only doing the block check if the height is above zero saves hashing // and saves a nontrivial amount of time during IBD. if hdb.blockHeight > 0 || block.ID() != types.GenesisID { hdb.blockHeight++ } else if hdb.blockHeight != 0 { // Sanity check - if the current block is the genesis block, the // hostdb height should be set to zero. hdb.log.Critical("Hostdb has detected a genesis block, but the height of the hostdbhostdb is set to ", hdb.blockHeight) hdb.blockHeight = 0 } } // Add hosts announced in blocks that were applied. for _, block := range cc.AppliedBlocks { for _, host := range findHostAnnouncements(block) { hdb.log.Debugln("Found a host in a host announcement:", host.NetAddress, host.PublicKey) hdb.insertHost(host) } } hdb.lastChange = cc.ID err := hdb.save() if err != nil { hdb.log.Println("Error saving hostdb:", err) } } Sia-1.0.4/modules/renter/hostdb/update_test.go000066400000000000000000000045761302033070500213370ustar00rootroot00000000000000package hostdb import ( "testing" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // makeSignedAnnouncement creates a []byte that contains an encoded and signed // host announcement for the given net address. func makeSignedAnnouncement(na modules.NetAddress) ([]byte, error) { sk, pk, err := crypto.GenerateKeyPair() if err != nil { return nil, err } spk := types.SiaPublicKey{ Algorithm: types.SignatureEd25519, Key: pk[:], } return modules.CreateAnnouncement(na, spk, sk) } // TestFindHostAnnouncements probes the findHostAnnouncements function func TestFindHostAnnouncements(t *testing.T) { annBytes, err := makeSignedAnnouncement("foo.com:1234") if err != nil { t.Fatal(err) } b := types.Block{ Transactions: []types.Transaction{ { ArbitraryData: [][]byte{annBytes}, }, }, } announcements := findHostAnnouncements(b) if len(announcements) != 1 { t.Error("host announcement not found in block") } // Try with an altered prefix b.Transactions[0].ArbitraryData[0][0]++ announcements = findHostAnnouncements(b) if len(announcements) != 0 { t.Error("host announcement found when there was an invalid prefix") } b.Transactions[0].ArbitraryData[0][0]-- // Try with an invalid host encoding. b.Transactions[0].ArbitraryData[0][17]++ announcements = findHostAnnouncements(b) if len(announcements) != 0 { t.Error("host announcement found when there was an invalid encoding of a host announcement") } } // TestReceiveConsensusSetUpdate probes the ReveiveConsensusSetUpdate method of // the hostdb type. func TestReceiveConsensusSetUpdate(t *testing.T) { // create hostdb hdb := bareHostDB() hdb.persist = &memPersist{} // Put a host announcement into a block. annBytes, err := makeSignedAnnouncement("foo.com:1234") if err != nil { t.Fatal(err) } cc := modules.ConsensusChange{ AppliedBlocks: []types.Block{{ Transactions: []types.Transaction{{ ArbitraryData: [][]byte{annBytes}, }}, }}, } // call ProcessConsensusChange hdb.ProcessConsensusChange(cc) // host should be sent to scanPool select { case <-time.After(time.Second): t.Fatal("announcement not seen") case <-hdb.scanPool: } // Check that there is now a host in the hostdb. if len(hdb.AllHosts()) != 1 { t.Fatal("hostdb should have a host after getting a host announcement transcation") } } Sia-1.0.4/modules/renter/hostdb/weightedlist.go000066400000000000000000000145651302033070500215110ustar00rootroot00000000000000package hostdb // weightedlist.go manages a weighted list of nodes that can be queried // randomly. The functions for inserting, removing, and fetching nodes from the // list are housed in this file. import ( "crypto/rand" "errors" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( errOverweight = errors.New("requested a too-heavy weight") ) // hostNode is the node of an unsorted, balanced, weighted binary tree. When // inserting elements, elements are inserted on the side of the tree with the // fewest elements. When removing, the node is just made empty but the tree is // not reorganized. The size of the tree will never decrease, but it will also // not increase unless it has more entries than it has ever had before. type hostNode struct { parent *hostNode count int // Cumulative count of this node and all children. // Currently the only weight supported is priceWeight. Eventually, support // will be added for multiple tunable types of weight. The different // weights all represent the cumulative weight of this node and all // children. weight types.Currency left *hostNode right *hostNode taken bool // Indicates whether there is an active host at this node or not. hostEntry *hostEntry } // createNode makes a new node the fill a host entry. func createNode(parent *hostNode, entry *hostEntry) *hostNode { return &hostNode{ parent: parent, weight: entry.Weight, count: 1, taken: true, hostEntry: entry, } } // nodeAtWeight grabs an element in the tree that appears at the given weight. // Though the tree has an arbitrary sorting, a sufficiently random weight will // pull a random element. The tree is searched through in a post-ordered way. func (hn *hostNode) nodeAtWeight(weight types.Currency) (*hostNode, error) { // Sanity check - weight must be less than the total weight of the tree. if weight.Cmp(hn.weight) > 0 { return nil, errOverweight } // Check if the left or right child should be returned. if hn.left != nil { if weight.Cmp(hn.left.weight) < 0 { return hn.left.nodeAtWeight(weight) } weight = weight.Sub(hn.left.weight) // Search from 0th index of right side. } if hn.right != nil && weight.Cmp(hn.right.weight) < 0 { return hn.right.nodeAtWeight(weight) } // Sanity check if build.DEBUG && !hn.taken { build.Critical("nodeAtWeight should not be returning a nil entry") } // Return the root entry. return hn, nil } // recursiveInsert is a recursive function for adding a hostNode to an existing tree // of hostNodes. The first call should always be on hostdb.hostTree. Running // time of recursiveInsert is log(n) in the maximum number of elements that have // ever been in the tree. func (hn *hostNode) recursiveInsert(entry *hostEntry) (nodesAdded int, newNode *hostNode) { hn.weight = hn.weight.Add(entry.Weight) // If the current node is empty, add the entry but don't increase the // count. if !hn.taken { hn.taken = true hn.hostEntry = entry newNode = hn return } // Insert the element into the lest populated side. if hn.left == nil { hn.left = createNode(hn, entry) nodesAdded = 1 newNode = hn.left } else if hn.right == nil { hn.right = createNode(hn, entry) nodesAdded = 1 newNode = hn.right } else if hn.left.count <= hn.right.count { nodesAdded, newNode = hn.left.recursiveInsert(entry) } else { nodesAdded, newNode = hn.right.recursiveInsert(entry) } hn.count += nodesAdded return } // remove takes a node and removes it from the tree by climbing through the // list of parents. remove does not delete nodes. func (hn *hostNode) removeNode() { hn.weight = hn.weight.Sub(hn.hostEntry.Weight) hn.taken = false current := hn.parent for current != nil { current.weight = current.weight.Sub(hn.hostEntry.Weight) current = current.parent } } // insertNode inserts a host entry into the host tree, removing // any conflicts. The host settings are assumed to be correct. Though hosts // with 0 weight will never be selected, they are accepted into the tree. func (hdb *HostDB) insertNode(entry *hostEntry) { // If there's already a host of the same id, remove that host. priorEntry, exists := hdb.activeHosts[entry.NetAddress] if exists { priorEntry.removeNode() } // Insert the updated entry into the host tree. if hdb.hostTree == nil { hdb.hostTree = createNode(nil, entry) hdb.activeHosts[entry.NetAddress] = hdb.hostTree } else { _, hostNode := hdb.hostTree.recursiveInsert(entry) hdb.activeHosts[entry.NetAddress] = hostNode } } // isEmpty returns whether the hostTree contains no entries. func (hdb *HostDB) isEmpty() bool { return hdb.hostTree == nil || hdb.hostTree.weight.IsZero() } // RandomHosts will pull up to 'n' random hosts from the hostdb. There will be // no repeats, but the length of the slice returned may be less than 'n', and // may even be 0. The hosts that get returned first have the higher priority. // Hosts specified in 'ignore' will not be considered; pass 'nil' if no // blacklist is desired. func (hdb *HostDB) RandomHosts(n int, ignore []modules.NetAddress) (hosts []modules.HostDBEntry) { hdb.mu.Lock() defer hdb.mu.Unlock() if hdb.isEmpty() { return } // These will be restored after selection is finished. var removedEntries []*hostEntry // Remove hosts that we want to ignore. for _, addr := range ignore { node, exists := hdb.activeHosts[addr] if !exists { continue } node.removeNode() delete(hdb.activeHosts, addr) removedEntries = append(removedEntries, node.hostEntry) } // Pick a host, remove it from the tree, and repeat until we have n hosts // or the tree is empty. for len(hosts) < n && !hdb.isEmpty() { randWeight, err := rand.Int(rand.Reader, hdb.hostTree.weight.Big()) if err != nil { build.Critical("rand.Int is returning an error:", err) break } node, err := hdb.hostTree.nodeAtWeight(types.NewCurrency(randWeight)) if err != nil { build.Critical("nodeAtWeight is returning and error:", err) break } // Only return the host if they are accepting contracts. if node.hostEntry.HostDBEntry.AcceptingContracts { hosts = append(hosts, node.hostEntry.HostDBEntry) } removedEntries = append(removedEntries, node.hostEntry) node.removeNode() delete(hdb.activeHosts, node.hostEntry.NetAddress) } // Add back all of the entries that got removed. for i := range removedEntries { hdb.insertNode(removedEntries[i]) } return hosts } Sia-1.0.4/modules/renter/hostdb/weightedlist_consistency_test.go000066400000000000000000000066541302033070500251710ustar00rootroot00000000000000package hostdb import ( "crypto/rand" "errors" "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // repeatCheckHelper recursively goes through nodes in the host map and adds // them to the repeat maps. func (hn *hostNode) repeatCheckHelper(ipMap, pkMap map[string]struct{}) error { ipStr := string(hn.hostEntry.NetAddress) pkStr := crypto.HashObject(hn.hostEntry.PublicKey).String() _, exists := ipMap[ipStr] if exists && hn.taken { return errors.New("found a duplicate ip address in the hostdb: " + ipStr) } _, exists = pkMap[pkStr] if exists && hn.taken { return errors.New("found a duplicate pubkey in the hostdb: " + ipStr + " " + pkStr) } if hn.taken { ipMap[ipStr] = struct{}{} pkMap[pkStr] = struct{}{} } if hn.left != nil { err := hn.left.repeatCheckHelper(ipMap, pkMap) if err != nil { return err } } if hn.right != nil { err := hn.right.repeatCheckHelper(ipMap, pkMap) if err != nil { return err } } return nil } // repeatCheck will return an error if there are multiple hosts in the host // tree with the same IP address or same public key. func repeatCheck(hn *hostNode) error { if hn == nil { return nil } ipMap := make(map[string]struct{}) pkMap := make(map[string]struct{}) err := hn.repeatCheckHelper(ipMap, pkMap) if err != nil { return err } return nil } // uniformTreeVerification checks that everything makes sense in the tree given // the number of entries that the tree is supposed to have and also given that // every entropy has the same weight. func uniformTreeVerification(hdb *HostDB, numEntries int) error { // Check that the weight of the hostTree is what is expected. expectedWeight := hdb.hostTree.hostEntry.Weight.Mul64(uint64(numEntries)) if hdb.hostTree.weight.Cmp(expectedWeight) != 0 { return errors.New("expected weight is incorrect") } // Check that the length of activeHosts and the count of hostTree are // consistent. if len(hdb.activeHosts) != numEntries { return errors.New("unexpected number of active hosts") } // Select many random hosts and do naive statistical analysis on the // results. if !testing.Short() { // Pull a bunch of random hosts and count how many times we pull each // host. selectionMap := make(map[modules.NetAddress]int) expected := 100 for i := 0; i < expected*numEntries; i++ { entries := hdb.RandomHosts(1, nil) if len(entries) == 0 { return errors.New("no hosts") } selectionMap[entries[0].NetAddress]++ } // See if each host was selected enough times. errorBound := 64 // Pretty large, but will still detect if something is seriously wrong. for _, count := range selectionMap { if count < expected-errorBound || count > expected+errorBound { return errors.New("error bound was breached") } } } // Try removing an re-adding all hosts. var removedEntries []*hostEntry for { if hdb.hostTree.weight.IsZero() { break } randWeight, err := rand.Int(rand.Reader, hdb.hostTree.weight.Big()) if err != nil { break } node, err := hdb.hostTree.nodeAtWeight(types.NewCurrency(randWeight)) if err != nil { break } node.removeNode() delete(hdb.activeHosts, node.hostEntry.NetAddress) // remove the entry from the hostdb so it won't be selected as a // repeat. removedEntries = append(removedEntries, node.hostEntry) } for _, entry := range removedEntries { hdb.insertNode(entry) } return nil } Sia-1.0.4/modules/renter/hostdb/weightedlist_test.go000066400000000000000000000212311302033070500225340ustar00rootroot00000000000000package hostdb import ( "crypto/rand" "math/big" "strconv" "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // fakeAddr returns a modules.NetAddress to be used in a HostEntry. Such // addresses are needed in order to satisfy the HostDB's "1 host per IP" rule. func fakeAddr(n uint8) modules.NetAddress { return modules.NetAddress("127.0.0." + strconv.Itoa(int(n)) + ":1") } // TestWeightedList inserts and removes nodes in a semi-random manner and // verifies that the tree stays consistent through the adjustments. func TestWeightedList(t *testing.T) { if testing.Short() { t.SkipNow() } // Create a hostdb and 3 equal entries to insert. hdb := bareHostDB() // Create a bunch of host entries of equal weight. var dbe modules.HostDBEntry dbe.AcceptingContracts = true firstInsertions := 64 for i := 0; i < firstInsertions; i++ { dbe.NetAddress = fakeAddr(uint8(i)) entry := hostEntry{ HostDBEntry: dbe, Weight: types.NewCurrency64(10), } hdb.insertNode(&entry) } err := uniformTreeVerification(hdb, firstInsertions) if err != nil { t.Error(err) } // Remove a few hosts and check that the tree is still in order. removals := 12 // Keep a map of what we've removed so far. removedMap := make(map[uint8]struct{}) for i := 0; i < removals; i++ { // Try numbers until we roll a number that's not been removed yet. var randInt uint8 for { randBig, err := rand.Int(rand.Reader, big.NewInt(int64(firstInsertions))) if err != nil { t.Fatal(err) } randInt = uint8(randBig.Int64()) _, exists := removedMap[randInt] if !exists { break } } // Remove the entry and add it to the list of removed entries err := hdb.removeHost(fakeAddr(randInt)) if err != nil { t.Fatal(err) } removedMap[randInt] = struct{}{} } err = uniformTreeVerification(hdb, firstInsertions-removals) if err != nil { t.Error(err) } // Do some more insertions. secondInsertions := 64 for i := firstInsertions; i < firstInsertions+secondInsertions; i++ { dbe.NetAddress = fakeAddr(uint8(i)) entry := hostEntry{ HostDBEntry: dbe, Weight: types.NewCurrency64(10), } hdb.insertNode(&entry) } err = uniformTreeVerification(hdb, firstInsertions-removals+secondInsertions) if err != nil { t.Error(err) } } // TestVariedWeights runs broad statistical tests on selecting hosts with // multiple different weights. func TestVariedWeights(t *testing.T) { if testing.Short() { t.SkipNow() } hdb := &HostDB{ activeHosts: make(map[modules.NetAddress]*hostNode), allHosts: make(map[modules.NetAddress]*hostEntry), scanPool: make(chan *hostEntry, scanPoolSize), } // insert i hosts with the weights 0, 1, ..., i-1. 100e3 selections will be made // per weight added to the tree, the total number of selections necessary // will be tallied up as hosts are created. var dbe modules.HostDBEntry dbe.AcceptingContracts = true hostCount := 5 expectedPerWeight := int(10e3) selections := 0 for i := 0; i < hostCount; i++ { dbe.NetAddress = fakeAddr(uint8(i)) entry := hostEntry{ HostDBEntry: dbe, Weight: types.NewCurrency64(uint64(i)), } hdb.insertNode(&entry) selections += i * expectedPerWeight } // Perform many random selections, noting which host was selected each // time. selectionMap := make(map[string]int) for i := 0; i < selections; i++ { randEntry := hdb.RandomHosts(1, nil) if len(randEntry) == 0 { t.Fatal("no hosts!") } node, exists := hdb.activeHosts[randEntry[0].NetAddress] if !exists { t.Fatal("can't find randomly selected node in tree") } selectionMap[node.hostEntry.Weight.String()]++ } // Check that each host was selected an expected number of times. An error // will be reported if the host of 0 weight is ever selected. acceptableError := 0.2 for weight, timesSelected := range selectionMap { intWeight, err := strconv.Atoi(weight) if err != nil { t.Fatal(err) } expectedSelected := float64(intWeight * expectedPerWeight) if float64(expectedSelected)*acceptableError > float64(timesSelected) || float64(expectedSelected)/acceptableError < float64(timesSelected) { t.Error("weighted list not selecting in a uniform distribution based on weight") t.Error(expectedSelected) t.Error(timesSelected) } } } // TestRepeatInsert inserts 2 hosts with the same address. func TestRepeatInsert(t *testing.T) { if testing.Short() { t.SkipNow() } hdb := &HostDB{ activeHosts: make(map[modules.NetAddress]*hostNode), allHosts: make(map[modules.NetAddress]*hostEntry), scanPool: make(chan *hostEntry, scanPoolSize), } var dbe modules.HostDBEntry dbe.NetAddress = fakeAddr(0) entry1 := hostEntry{ HostDBEntry: dbe, Weight: types.NewCurrency64(1), } entry2 := entry1 hdb.insertNode(&entry1) entry2.Weight = types.NewCurrency64(100) hdb.insertNode(&entry2) if len(hdb.activeHosts) != 1 { t.Error("insterting the same entry twice should result in only 1 entry in the hostdb") } } // TestNodeAtWeight tests the nodeAtWeight method. func TestNodeAtWeight(t *testing.T) { // create hostTree h1 := new(hostEntry) h1.NetAddress = "foo" h1.Weight = baseWeight ht := createNode(nil, h1) // overweight _, err := ht.nodeAtWeight(baseWeight.Mul64(2)) if err != errOverweight { t.Errorf("expected %v, got %v", errOverweight, err) } h, err := ht.nodeAtWeight(baseWeight) if err != nil { t.Error(err) } else if h.hostEntry != h1 { t.Errorf("nodeAtWeight returned wrong node: expected %v, got %v", h1, h.hostEntry) } } // TestRandomHosts probes the RandomHosts function. func TestRandomHosts(t *testing.T) { // Create the hostdb. hdb := bareHostDB() // Empty. if hosts := hdb.RandomHosts(1, nil); len(hosts) != 0 { t.Errorf("empty hostdb returns %v hosts: %v", len(hosts), hosts) } // Insert 3 hosts to be selected. var dbe modules.HostDBEntry dbe.NetAddress = fakeAddr(1) dbe.AcceptingContracts = true entry1 := hostEntry{ HostDBEntry: dbe, Weight: types.NewCurrency64(1), } dbe.NetAddress = fakeAddr(2) entry2 := hostEntry{ HostDBEntry: dbe, Weight: types.NewCurrency64(2), } dbe.NetAddress = fakeAddr(3) entry3 := hostEntry{ HostDBEntry: dbe, Weight: types.NewCurrency64(3), } hdb.insertNode(&entry1) hdb.insertNode(&entry2) hdb.insertNode(&entry3) if len(hdb.activeHosts) != 3 { t.Error("wrong number of hosts") } if hdb.hostTree.weight.Cmp64(6) != 0 { t.Error("unexpected weight at initialization") t.Error(hdb.hostTree.weight) } // Grab 1 random host. randHosts := hdb.RandomHosts(1, nil) if len(randHosts) != 1 { t.Error("didn't get 1 hosts") } // Grab 2 random hosts. randHosts = hdb.RandomHosts(2, nil) if len(randHosts) != 2 { t.Error("didn't get 2 hosts") } if randHosts[0].NetAddress == randHosts[1].NetAddress { t.Error("doubled up") } // Grab 3 random hosts. randHosts = hdb.RandomHosts(3, nil) if len(randHosts) != 3 { t.Error("didn't get 3 hosts") } if randHosts[0].NetAddress == randHosts[1].NetAddress || randHosts[0].NetAddress == randHosts[2].NetAddress || randHosts[1].NetAddress == randHosts[2].NetAddress { t.Error("doubled up") } // Grab 4 random hosts. 3 should be returned. randHosts = hdb.RandomHosts(4, nil) if len(randHosts) != 3 { t.Error("didn't get 3 hosts") } if randHosts[0].NetAddress == randHosts[1].NetAddress || randHosts[0].NetAddress == randHosts[2].NetAddress || randHosts[1].NetAddress == randHosts[2].NetAddress { t.Error("doubled up") } // Ask for 3 hosts that are not in randHosts. No hosts should be // returned. uniqueHosts := hdb.RandomHosts(3, []modules.NetAddress{ randHosts[0].NetAddress, randHosts[1].NetAddress, randHosts[2].NetAddress, }) if len(uniqueHosts) != 0 { t.Error("didn't get 0 hosts") } // Ask for 3 hosts, blacklisting non-existent hosts. 3 should be returned. randHosts = hdb.RandomHosts(3, []modules.NetAddress{"foo", "bar", "baz"}) if len(randHosts) != 3 { t.Error("didn't get 3 hosts") } if randHosts[0].NetAddress == randHosts[1].NetAddress || randHosts[0].NetAddress == randHosts[2].NetAddress || randHosts[1].NetAddress == randHosts[2].NetAddress { t.Error("doubled up") } // entry4 should not every be returned by RandomHosts because it is not // accepting contracts. dbe.NetAddress = fakeAddr(4) dbe.AcceptingContracts = false entry4 := hostEntry{ HostDBEntry: dbe, Weight: types.NewCurrency64(4), } hdb.insertNode(&entry4) // Grab 4 random hosts. 3 should be returned. randHosts = hdb.RandomHosts(4, nil) if len(randHosts) != 3 { t.Error("didn't get 3 hosts") } if randHosts[0].NetAddress == randHosts[1].NetAddress || randHosts[0].NetAddress == randHosts[2].NetAddress || randHosts[1].NetAddress == randHosts[2].NetAddress { t.Error("doubled up") } } Sia-1.0.4/modules/renter/persist.go000066400000000000000000000240741302033070500172170ustar00rootroot00000000000000package renter import ( "bytes" "compress/gzip" "encoding/base64" "errors" "io" "os" "path/filepath" "strconv" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/types" ) const ( PersistFilename = "renter.json" ShareExtension = ".sia" logFile = modules.RenterDir + ".log" ) var ( ErrNoNicknames = errors.New("at least one nickname must be supplied") ErrNonShareSuffix = errors.New("suffix of file must be " + ShareExtension) ErrBadFile = errors.New("not a .sia file") ErrIncompatible = errors.New("file is not compatible with current version") shareHeader = [15]byte{'S', 'i', 'a', ' ', 'S', 'h', 'a', 'r', 'e', 'd', ' ', 'F', 'i', 'l', 'e'} shareVersion = "0.4" saveMetadata = persist.Metadata{ Header: "Renter Persistence", Version: "0.4", } ) // MarshalSia implements the encoding.SiaMarshaller interface, writing the // file data to w. func (f *file) MarshalSia(w io.Writer) error { enc := encoding.NewEncoder(w) // encode easy fields err := enc.EncodeAll( f.name, f.size, f.masterKey, f.pieceSize, f.mode, ) if err != nil { return err } // COMPATv0.4.3 - encode the bytesUploaded and chunksUploaded fields // TODO: the resulting .sia file may confuse old clients. err = enc.EncodeAll(f.pieceSize*f.numChunks()*uint64(f.erasureCode.NumPieces()), f.numChunks()) if err != nil { return err } // encode erasureCode switch code := f.erasureCode.(type) { case *rsCode: err = enc.EncodeAll( "Reed-Solomon", uint64(code.dataPieces), uint64(code.numPieces-code.dataPieces), ) if err != nil { return err } default: if build.DEBUG { panic("unknown erasure code") } return errors.New("unknown erasure code") } // encode contracts if err := enc.Encode(uint64(len(f.contracts))); err != nil { return err } for _, c := range f.contracts { if err := enc.Encode(c); err != nil { return err } } return nil } // UnmarshalSia implements the encoding.SiaUnmarshaller interface, // reconstructing a file from the encoded bytes read from r. func (f *file) UnmarshalSia(r io.Reader) error { dec := encoding.NewDecoder(r) // COMPATv0.4.3 - decode bytesUploaded and chunksUploaded into dummy vars. var bytesUploaded, chunksUploaded uint64 // decode easy fields err := dec.DecodeAll( &f.name, &f.size, &f.masterKey, &f.pieceSize, &f.mode, &bytesUploaded, &chunksUploaded, ) if err != nil { return err } // decode erasure coder var codeType string if err := dec.Decode(&codeType); err != nil { return err } switch codeType { case "Reed-Solomon": var nData, nParity uint64 err = dec.DecodeAll( &nData, &nParity, ) if err != nil { return err } rsc, err := NewRSCode(int(nData), int(nParity)) if err != nil { return err } f.erasureCode = rsc default: return errors.New("unrecognized erasure code type: " + codeType) } // decode contracts var nContracts uint64 if err := dec.Decode(&nContracts); err != nil { return err } f.contracts = make(map[types.FileContractID]fileContract) var contract fileContract for i := uint64(0); i < nContracts; i++ { if err := dec.Decode(&contract); err != nil { return err } f.contracts[contract.ID] = contract } return nil } // saveFile saves a file to the renter directory. func (r *Renter) saveFile(f *file) error { // Create directory structure specified in nickname. fullPath := filepath.Join(r.persistDir, f.name+ShareExtension) err := os.MkdirAll(filepath.Dir(fullPath), 0700) if err != nil { return err } // Open SafeFile handle. handle, err := persist.NewSafeFile(filepath.Join(r.persistDir, f.name+ShareExtension)) if err != nil { return err } defer handle.Close() // Write file data. err = shareFiles([]*file{f}, handle) if err != nil { return err } // Commit the SafeFile. return handle.Commit() } // save stores the current renter data to disk. func (r *Renter) save() error { data := struct { Tracking map[string]trackedFile }{r.tracking} return persist.SaveFile(saveMetadata, data, filepath.Join(r.persistDir, PersistFilename)) } // saveSync stores the current renter data to disk and then syncs to disk. func (r *Renter) saveSync() error { data := struct { Tracking map[string]trackedFile }{r.tracking} return persist.SaveFileSync(saveMetadata, data, filepath.Join(r.persistDir, PersistFilename)) } // load fetches the saved renter data from disk. func (r *Renter) load() error { // Recursively load all files found in renter directory. Errors // encountered during loading are logged, but are not considered fatal. err := filepath.Walk(r.persistDir, func(path string, info os.FileInfo, err error) error { // This error is non-nil if filepath.Walk couldn't stat a file or // folder. This generally indicates a serious error. if err != nil { return err } // Skip folders and non-sia files. if info.IsDir() || filepath.Ext(path) != ShareExtension { return nil } // Open the file. file, err := os.Open(path) if err != nil { r.log.Println("ERROR: could not open .sia file:", err) return nil } defer file.Close() // Load the file contents into the renter. _, err = r.loadSharedFiles(file) if err != nil { r.log.Println("ERROR: could not load .sia file:", err) return nil } return nil }) if err != nil { return err } // Load contracts, repair set, and entropy. data := struct { Tracking map[string]trackedFile Repairing map[string]string // COMPATv0.4.8 }{} err = persist.LoadFile(saveMetadata, &data, filepath.Join(r.persistDir, PersistFilename)) if err != nil { return err } if data.Tracking != nil { r.tracking = data.Tracking } return nil } // shareFiles writes the specified files to w. First a header is written, // followed by the gzipped concatenation of each file. func shareFiles(files []*file, w io.Writer) error { // Write header. err := encoding.NewEncoder(w).EncodeAll( shareHeader, shareVersion, uint64(len(files)), ) if err != nil { return err } // Create compressor. zip, _ := gzip.NewWriterLevel(w, gzip.BestCompression) enc := encoding.NewEncoder(zip) // Encode each file. for _, f := range files { err = enc.Encode(f) if err != nil { return err } } return zip.Close() } // ShareFile saves the specified files to shareDest. func (r *Renter) ShareFiles(nicknames []string, shareDest string) error { lockID := r.mu.RLock() defer r.mu.RUnlock(lockID) // TODO: consider just appending the proper extension. if filepath.Ext(shareDest) != ShareExtension { return ErrNonShareSuffix } handle, err := os.Create(shareDest) if err != nil { return err } defer handle.Close() // Load files from renter. files := make([]*file, len(nicknames)) for i, name := range nicknames { f, exists := r.files[name] if !exists { return ErrUnknownPath } files[i] = f } err = shareFiles(files, handle) if err != nil { os.Remove(shareDest) return err } return nil } // ShareFilesAscii returns the specified files in ASCII format. func (r *Renter) ShareFilesAscii(nicknames []string) (string, error) { lockID := r.mu.RLock() defer r.mu.RUnlock(lockID) // Load files from renter. files := make([]*file, len(nicknames)) for i, name := range nicknames { f, exists := r.files[name] if !exists { return "", ErrUnknownPath } files[i] = f } buf := new(bytes.Buffer) err := shareFiles(files, base64.NewEncoder(base64.URLEncoding, buf)) if err != nil { return "", err } return buf.String(), nil } // loadSharedFiles reads .sia data from reader and registers the contained // files in the renter. It returns the nicknames of the loaded files. func (r *Renter) loadSharedFiles(reader io.Reader) ([]string, error) { // read header var header [15]byte var version string var numFiles uint64 err := encoding.NewDecoder(reader).DecodeAll( &header, &version, &numFiles, ) if err != nil { return nil, err } else if header != shareHeader { return nil, ErrBadFile } else if version != shareVersion { return nil, ErrIncompatible } // Create decompressor. unzip, err := gzip.NewReader(reader) if err != nil { return nil, err } dec := encoding.NewDecoder(unzip) // Read each file. files := make([]*file, numFiles) for i := range files { files[i] = new(file) err := dec.Decode(files[i]) if err != nil { return nil, err } // Make sure the file's name does not conflict with existing files. dupCount := 0 origName := files[i].name for { _, exists := r.files[files[i].name] if !exists { break } dupCount++ files[i].name = origName + "_" + strconv.Itoa(dupCount) } } // Add files to renter. names := make([]string, numFiles) for i, f := range files { r.files[f.name] = f names[i] = f.name } // Save the files. for _, f := range files { r.saveFile(f) } return names, nil } // initPersist handles all of the persistence initialization, such as creating // the persistence directory and starting the logger. func (r *Renter) initPersist() error { // Create the perist directory if it does not yet exist. err := os.MkdirAll(r.persistDir, 0700) if err != nil { return err } // Initialize the logger. r.log, err = persist.NewFileLogger(filepath.Join(r.persistDir, logFile)) if err != nil { return err } // Load the prior persistence structures. err = r.load() if err != nil && !os.IsNotExist(err) { return err } return nil } // LoadSharedFiles loads a .sia file into the renter. It returns the nicknames // of the loaded files. func (r *Renter) LoadSharedFiles(filename string) ([]string, error) { lockID := r.mu.Lock() defer r.mu.Unlock(lockID) file, err := os.Open(filename) if err != nil { return nil, err } defer file.Close() return r.loadSharedFiles(file) } // LoadSharedFilesAscii loads an ASCII-encoded .sia file into the renter. It // returns the nicknames of the loaded files. func (r *Renter) LoadSharedFilesAscii(asciiSia string) ([]string, error) { lockID := r.mu.Lock() defer r.mu.Unlock(lockID) dec := base64.NewDecoder(base64.URLEncoding, bytes.NewBufferString(asciiSia)) return r.loadSharedFiles(dec) } Sia-1.0.4/modules/renter/persist_test.go000066400000000000000000000175741302033070500202650ustar00rootroot00000000000000package renter import ( "bytes" "fmt" "os" "path/filepath" "strconv" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" ) // newTestingFile initializes a file object with random parameters. func newTestingFile() *file { key, _ := crypto.GenerateTwofishKey() data, _ := crypto.RandBytes(8) nData, _ := crypto.RandIntn(10) nParity, _ := crypto.RandIntn(10) rsc, _ := NewRSCode(nData+1, nParity+1) return &file{ name: "testfile-" + strconv.Itoa(int(data[0])), size: encoding.DecUint64(data[1:5]), masterKey: key, erasureCode: rsc, pieceSize: encoding.DecUint64(data[6:8]), } } // equalFiles is a helper function that compares two files for equality. func equalFiles(f1, f2 *file) error { if f1 == nil || f2 == nil { return fmt.Errorf("one or both files are nil") } if f1.name != f2.name { return fmt.Errorf("names do not match: %v %v", f1.name, f2.name) } if f1.size != f2.size { return fmt.Errorf("sizes do not match: %v %v", f1.size, f2.size) } if f1.masterKey != f2.masterKey { return fmt.Errorf("keys do not match: %v %v", f1.masterKey, f2.masterKey) } if f1.pieceSize != f2.pieceSize { return fmt.Errorf("pieceSizes do not match: %v %v", f1.pieceSize, f2.pieceSize) } return nil } // TestFileMarshalling tests the MarshalSia and UnmarshalSia functions of the // file type. func TestFileMarshalling(t *testing.T) { savedFile := newTestingFile() buf := new(bytes.Buffer) savedFile.MarshalSia(buf) loadedFile := new(file) err := loadedFile.UnmarshalSia(buf) if err != nil { t.Fatal(err) } err = equalFiles(savedFile, loadedFile) if err != nil { t.Fatal(err) } } // TestFileShareLoad tests the sharing/loading functions of the renter. func TestFileShareLoad(t *testing.T) { if testing.Short() { t.SkipNow() } rt, err := newRenterTester("TestRenterShareLoad") if err != nil { t.Fatal(err) } defer rt.Close() // Create a file and add it to the renter. savedFile := newTestingFile() rt.renter.files[savedFile.name] = savedFile // Share .sia file to disk. path := filepath.Join(build.SiaTestingDir, "renter", "TestRenterShareLoad", "test.sia") err = rt.renter.ShareFiles([]string{savedFile.name}, path) if err != nil { t.Fatal(err) } // Remove the file from the renter. delete(rt.renter.files, savedFile.name) // Load the .sia file back into the renter. names, err := rt.renter.LoadSharedFiles(path) if err != nil { t.Fatal(err) } if len(names) != 1 || names[0] != savedFile.name { t.Fatal("nickname not loaded properly:", names) } err = equalFiles(rt.renter.files[savedFile.name], savedFile) if err != nil { t.Fatal(err) } // Share and load multiple files. savedFile2 := newTestingFile() rt.renter.files[savedFile2.name] = savedFile2 path = filepath.Join(build.SiaTestingDir, "renter", "TestRenterShareLoad", "test2.sia") err = rt.renter.ShareFiles([]string{savedFile.name, savedFile2.name}, path) if err != nil { t.Fatal(err) } // Remove the files from the renter. delete(rt.renter.files, savedFile.name) delete(rt.renter.files, savedFile2.name) names, err = rt.renter.LoadSharedFiles(path) if err != nil { t.Fatal(nil) } if len(names) != 2 || (names[0] != savedFile2.name && names[1] != savedFile2.name) { t.Fatal("nicknames not loaded properly:", names) } err = equalFiles(rt.renter.files[savedFile.name], savedFile) if err != nil { t.Fatal(err) } err = equalFiles(rt.renter.files[savedFile2.name], savedFile2) if err != nil { t.Fatal(err) } } // TestFileShareLoadASCII tests the ASCII sharing/loading functions. func TestFileShareLoadASCII(t *testing.T) { if testing.Short() { t.SkipNow() } rt, err := newRenterTester("TestRenterShareLoadASCII") if err != nil { t.Fatal(err) } defer rt.Close() // Create a file and add it to the renter. savedFile := newTestingFile() rt.renter.files[savedFile.name] = savedFile ascii, err := rt.renter.ShareFilesAscii([]string{savedFile.name}) if err != nil { t.Fatal(err) } // Remove the file from the renter. delete(rt.renter.files, savedFile.name) names, err := rt.renter.LoadSharedFilesAscii(ascii) if err != nil { t.Fatal(err) } if len(names) != 1 || names[0] != savedFile.name { t.Fatal("nickname not loaded properly") } err = equalFiles(rt.renter.files[savedFile.name], savedFile) if err != nil { t.Fatal(err) } } // TestRenterSaveLoad probes the save and load methods of the renter type. func TestRenterSaveLoad(t *testing.T) { if testing.Short() { t.SkipNow() } rt, err := newRenterTester("TestRenterSaveLoad") if err != nil { t.Fatal(err) } defer rt.Close() // Create and save some files var f1, f2, f3 *file f1 = newTestingFile() f2 = newTestingFile() f3 = newTestingFile() // names must not conflict for f2.name == f1.name || f2.name == f3.name { f2 = newTestingFile() } for f3.name == f1.name || f3.name == f2.name { f3 = newTestingFile() } rt.renter.saveFile(f1) rt.renter.saveFile(f2) rt.renter.saveFile(f3) err = rt.renter.save() // save metadata if err != nil { t.Fatal(err) } // load should now load the files into memory. id := rt.renter.mu.Lock() err = rt.renter.load() rt.renter.mu.Unlock(id) if err != nil && !os.IsNotExist(err) { t.Fatal(err) } if err := equalFiles(f1, rt.renter.files[f1.name]); err != nil { t.Fatal(err) } if err := equalFiles(f2, rt.renter.files[f2.name]); err != nil { t.Fatal(err) } if err := equalFiles(f3, rt.renter.files[f3.name]); err != nil { t.Fatal(err) } } // TestRenterPaths checks that the renter properly handles nicknames // containing the path separator ("/"). func TestRenterPaths(t *testing.T) { if testing.Short() { t.SkipNow() } rt, err := newRenterTester("TestRenterPaths") if err != nil { t.Fatal(err) } defer rt.Close() // Create and save some files. // The result of saving these files should be a directory containing: // foo.sia // foo/bar.sia // foo/bar/baz.sia f1 := newTestingFile() f1.name = "foo" f2 := newTestingFile() f2.name = "foo/bar" f3 := newTestingFile() f3.name = "foo/bar/baz" rt.renter.saveFile(f1) rt.renter.saveFile(f2) rt.renter.saveFile(f3) // Load the files into the renter. id := rt.renter.mu.Lock() err = rt.renter.load() rt.renter.mu.Unlock(id) if err != nil && !os.IsNotExist(err) { t.Fatal(err) } // Check that the files were loaded properly. if err := equalFiles(f1, rt.renter.files[f1.name]); err != nil { t.Fatal(err) } if err := equalFiles(f2, rt.renter.files[f2.name]); err != nil { t.Fatal(err) } if err := equalFiles(f3, rt.renter.files[f3.name]); err != nil { t.Fatal(err) } // To confirm that the file structure was preserved, we walk the renter // folder and emit the name of each .sia file encountered (filepath.Walk // is deterministic; it orders the files lexically). var walkStr string filepath.Walk(rt.renter.persistDir, func(path string, _ os.FileInfo, _ error) error { // capture only .sia files if filepath.Ext(path) != ".sia" { return nil } rel, _ := filepath.Rel(rt.renter.persistDir, path) // strip testdir prefix walkStr += rel return nil }) // walk will descend into foo/bar/, reading baz, bar, and finally foo expWalkStr := (f3.name + ".sia") + (f2.name + ".sia") + (f1.name + ".sia") if filepath.ToSlash(walkStr) != expWalkStr { t.Fatalf("Bad walk string: expected %v, got %v", expWalkStr, walkStr) } } // TestSiafileCompatibility tests that the renter is able to load v0.4.8 .sia files. func TestSiafileCompatibility(t *testing.T) { if testing.Short() { t.SkipNow() } rt, err := newRenterTester("TestSiafileCompatibility") if err != nil { t.Fatal(err) } defer rt.Close() // Load the compatibility file into the renter. path := filepath.Join("..", "..", "compatibility", "siafile_v0.4.8.sia") names, err := rt.renter.LoadSharedFiles(path) if err != nil { t.Fatal(err) } if len(names) != 1 || names[0] != "testfile-183" { t.Fatal("nickname not loaded properly:", names) } } Sia-1.0.4/modules/renter/pool.go000066400000000000000000000060631302033070500164750ustar00rootroot00000000000000package renter import ( "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/renter/contractor" ) // A hostPool is a collection of active host connections, in the form of // Editors. The renter uses a hostPool to prevent connecting to the same host // more than once. This is more efficient, and also makes it easier to // serialize contract revisions. type hostPool struct { hosts []contractor.Editor blacklist []modules.NetAddress hostContractor hostContractor hdb hostDB } // Close closes all of the hostPool's open host connections. func (p *hostPool) Close() error { for _, h := range p.hosts { h.Close() } return nil } // add adds a contract's host to the hostPool and returns it as an Editor. func (p *hostPool) add(contract modules.RenterContract) (contractor.Editor, error) { for _, h := range p.hosts { if h.Address() == contract.NetAddress { return h, nil } } hu, err := p.hostContractor.Editor(contract.ID) if err != nil { p.blacklist = append(p.blacklist, contract.NetAddress) return nil, err } p.hosts = append(p.hosts, hu) return hu, nil } // remove disconnects from a host and adds it to the blacklist. func (p *hostPool) remove(addr modules.NetAddress) { for i, h := range p.hosts { if h.Address() == addr { h.Close() p.hosts = append(p.hosts[:i], p.hosts[i+1:]...) p.blacklist = append(p.blacklist, addr) return } } build.Critical("could not remove host from pool: no record of host", addr) } // uniqueHosts will return up to 'n' unique hosts that are not in 'exclude'. // The pool draws from its set of active connections first, and then negotiates // new contracts if more hosts are required. Note that this latter case // requires network I/O, so the caller should always assume that uniqueHosts // will block. func (p *hostPool) uniqueHosts(n int, exclude []modules.NetAddress) (hosts []contractor.Editor) { if n == 0 { return } // convert slice to map for easier lookups excludeSet := make(map[modules.NetAddress]struct{}) for _, ip := range exclude { excludeSet[ip] = struct{}{} } // First reuse existing connections. for _, h := range p.hosts { if _, ok := excludeSet[h.Address()]; ok { continue } hosts = append(hosts, h) if len(hosts) >= n { return hosts } } // Extend the exclude set with the pool's blacklist, and the hosts we're // already connected to. for _, ip := range p.blacklist { excludeSet[ip] = struct{}{} } for _, h := range p.hosts { excludeSet[h.Address()] = struct{}{} } // Next try to reuse existing contracts. for _, contract := range p.hostContractor.Contracts() { if _, ok := excludeSet[contract.NetAddress]; ok { continue } hu, err := p.add(contract) if err != nil { continue } hosts = append(hosts, hu) excludeSet[hu.Address()] = struct{}{} if len(hosts) >= n { break } } return hosts } // newHostPool returns an empty hostPool. func (r *Renter) newHostPool() *hostPool { return &hostPool{ hostContractor: r.hostContractor, hdb: r.hostDB, } } Sia-1.0.4/modules/renter/proto/000077500000000000000000000000001302033070500163335ustar00rootroot00000000000000Sia-1.0.4/modules/renter/proto/downloader.go000066400000000000000000000117461302033070500210310ustar00rootroot00000000000000package proto import ( "errors" "net" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // A Downloader retrieves sectors by calling the download RPC on a host. // Downloaders are NOT thread- safe; calls to Sector must be serialized. type Downloader struct { host modules.HostDBEntry contract modules.RenterContract // updated after each revision conn net.Conn SaveFn revisionSaver // metrics DownloadSpending types.Currency } // Sector retrieves the sector with the specified Merkle root, and revises // the underlying contract to pay the host proportionally to the data // retrieve. func (hd *Downloader) Sector(root crypto.Hash) (modules.RenterContract, []byte, error) { extendDeadline(hd.conn, modules.NegotiateDownloadTime) defer extendDeadline(hd.conn, time.Hour) // reset deadline when finished // calculate price sectorPrice := hd.host.DownloadBandwidthPrice.Mul64(modules.SectorSize) if hd.contract.RenterFunds().Cmp(sectorPrice) < 0 { return modules.RenterContract{}, nil, errors.New("contract has insufficient funds to support download") } // create the download revision rev := newDownloadRevision(hd.contract.LastRevision, sectorPrice) // initiate download by confirming host settings if err := startDownload(hd.conn, hd.host); err != nil { return modules.RenterContract{}, nil, err } // Before we continue, save the revision. Unexpected termination (e.g. // power failure) during the signature transfer leaves in an ambiguous // state: the host may or may not have received the signature, and thus // may report either revision as being the most recent. To mitigate this, // we save the old revision as a fallback. if hd.SaveFn != nil { if err := hd.SaveFn(rev, hd.contract.MerkleRoots); err != nil { return modules.RenterContract{}, nil, err } } // send download action err := encoding.WriteObject(hd.conn, []modules.DownloadAction{{ MerkleRoot: root, Offset: 0, Length: modules.SectorSize, }}) if err != nil { return modules.RenterContract{}, nil, err } // send the revision to the host for approval signedTxn, err := negotiateRevision(hd.conn, rev, hd.contract.SecretKey) if err == modules.ErrStopResponse { // if host gracefully closed, close our connection as well; this will // cause the next download to fail. However, we must delay closing // until we've finished downloading the sector. defer hd.conn.Close() } else if err != nil { return modules.RenterContract{}, nil, err } // read sector data, completing one iteration of the download loop var sectors [][]byte if err := encoding.ReadObject(hd.conn, §ors, modules.SectorSize+16); err != nil { return modules.RenterContract{}, nil, err } else if len(sectors) != 1 { return modules.RenterContract{}, nil, errors.New("host did not send enough sectors") } sector := sectors[0] if uint64(len(sector)) != modules.SectorSize { return modules.RenterContract{}, nil, errors.New("host did not send enough sector data") } else if crypto.MerkleRoot(sector) != root { return modules.RenterContract{}, nil, errors.New("host sent bad sector data") } // update contract and metrics hd.contract.LastRevision = rev hd.contract.LastRevisionTxn = signedTxn hd.DownloadSpending = hd.DownloadSpending.Add(sectorPrice) return hd.contract, sector, nil } // Close cleanly terminates the download loop with the host and closes the // connection. func (hd *Downloader) Close() error { extendDeadline(hd.conn, modules.NegotiateSettingsTime) // don't care about these errors _, _ = verifySettings(hd.conn, hd.host) _ = modules.WriteNegotiationStop(hd.conn) return hd.conn.Close() } // NewDownloader initiates the download request loop with a host, and returns a // Downloader. func NewDownloader(host modules.HostDBEntry, contract modules.RenterContract) (*Downloader, error) { // check that contract has enough value to support a download if len(contract.LastRevision.NewValidProofOutputs) != 2 { return nil, errors.New("invalid contract") } sectorPrice := host.DownloadBandwidthPrice.Mul64(modules.SectorSize) if contract.RenterFunds().Cmp(sectorPrice) < 0 { return nil, errors.New("contract has insufficient funds to support download") } // initiate download loop conn, err := net.DialTimeout("tcp", string(contract.NetAddress), 15*time.Second) if err != nil { return nil, err } // allot 2 minutes for RPC request + revision exchange extendDeadline(conn, modules.NegotiateRecentRevisionTime) defer extendDeadline(conn, time.Hour) if err := encoding.WriteObject(conn, modules.RPCDownload); err != nil { conn.Close() return nil, errors.New("couldn't initiate RPC: " + err.Error()) } if err := verifyRecentRevision(conn, contract); err != nil { conn.Close() // TODO: close gracefully if host has entered revision loop return nil, err } // the host is now ready to accept revisions return &Downloader{ contract: contract, host: host, conn: conn, }, nil } Sia-1.0.4/modules/renter/proto/editor.go000066400000000000000000000217441302033070500201600ustar00rootroot00000000000000package proto import ( "errors" "net" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( // sectorHeight is the height of a Merkle tree that covers a single // sector. It is log2(modules.SectorSize / crypto.SegmentSize) sectorHeight = func() uint64 { height := uint64(0) for 1< 0 { sectorPrice = sectorPrice.MulFloat(1.002) sectorCollateral = sectorCollateral.MulFloat(0.998) } // calculate the new Merkle root sectorRoot := crypto.MerkleRoot(data) newRoots := append(he.contract.MerkleRoots, sectorRoot) merkleRoot := cachedMerkleRoot(newRoots) // create the action and revision actions := []modules.RevisionAction{{ Type: modules.ActionInsert, SectorIndex: uint64(len(he.contract.MerkleRoots)), Data: data, }} rev := newUploadRevision(he.contract.LastRevision, merkleRoot, sectorPrice, sectorCollateral) // run the revision iteration if err := he.runRevisionIteration(actions, rev, newRoots); err != nil { return modules.RenterContract{}, crypto.Hash{}, err } // update metrics he.StorageSpending = he.StorageSpending.Add(sectorStoragePrice) he.UploadSpending = he.UploadSpending.Add(sectorBandwidthPrice) return he.contract, sectorRoot, nil } // Delete negotiates a revision that removes a sector from a file contract. func (he *Editor) Delete(root crypto.Hash) (modules.RenterContract, error) { // allot 2 minutes for this exchange extendDeadline(he.conn, 120*time.Second) defer extendDeadline(he.conn, time.Hour) // reset deadline // calculate the new Merkle root newRoots := make([]crypto.Hash, 0, len(he.contract.MerkleRoots)) index := -1 for i, h := range he.contract.MerkleRoots { if h == root { index = i } else { newRoots = append(newRoots, h) } } if index == -1 { return modules.RenterContract{}, errors.New("no record of that sector root") } merkleRoot := cachedMerkleRoot(newRoots) // create the action and accompanying revision actions := []modules.RevisionAction{{ Type: modules.ActionDelete, SectorIndex: uint64(index), }} rev := newDeleteRevision(he.contract.LastRevision, merkleRoot) // run the revision iteration if err := he.runRevisionIteration(actions, rev, newRoots); err != nil { return modules.RenterContract{}, err } return he.contract, nil } // Modify negotiates a revision that edits a sector in a file contract. func (he *Editor) Modify(oldRoot, newRoot crypto.Hash, offset uint64, newData []byte) (modules.RenterContract, error) { // allot 10 minutes for this exchange; sufficient to transfer 4 MB over 50 kbps extendDeadline(he.conn, modules.NegotiateFileContractRevisionTime) defer extendDeadline(he.conn, time.Hour) // reset deadline // calculate price sectorBandwidthPrice := he.host.UploadBandwidthPrice.Mul64(uint64(len(newData))) if he.contract.RenterFunds().Cmp(sectorBandwidthPrice) < 0 { return modules.RenterContract{}, errors.New("contract has insufficient funds to support modification") } // calculate the new Merkle root newRoots := make([]crypto.Hash, len(he.contract.MerkleRoots)) index := -1 for i, h := range he.contract.MerkleRoots { if h == oldRoot { index = i newRoots[i] = newRoot } else { newRoots[i] = h } } if index == -1 { return modules.RenterContract{}, errors.New("no record of that sector root") } merkleRoot := cachedMerkleRoot(newRoots) // create the action and revision actions := []modules.RevisionAction{{ Type: modules.ActionModify, SectorIndex: uint64(index), Offset: offset, Data: newData, }} rev := newModifyRevision(he.contract.LastRevision, merkleRoot, sectorBandwidthPrice) // run the revision iteration if err := he.runRevisionIteration(actions, rev, newRoots); err != nil { return modules.RenterContract{}, err } // update metrics he.UploadSpending = he.UploadSpending.Add(sectorBandwidthPrice) return he.contract, nil } // NewEditor initiates the contract revision process with a host, and returns // an Editor. func NewEditor(host modules.HostDBEntry, contract modules.RenterContract, currentHeight types.BlockHeight) (*Editor, error) { // check that contract has enough value to support an upload if len(contract.LastRevision.NewValidProofOutputs) != 2 { return nil, errors.New("invalid contract") } // initiate revision loop conn, err := net.DialTimeout("tcp", string(contract.NetAddress), 15*time.Second) if err != nil { return nil, err } // allot 2 minutes for RPC request + revision exchange extendDeadline(conn, modules.NegotiateRecentRevisionTime) defer extendDeadline(conn, time.Hour) if err := encoding.WriteObject(conn, modules.RPCReviseContract); err != nil { conn.Close() return nil, errors.New("couldn't initiate RPC: " + err.Error()) } if err := verifyRecentRevision(conn, contract); err != nil { conn.Close() // TODO: close gracefully if host has entered revision loop return nil, err } // the host is now ready to accept revisions return &Editor{ host: host, height: currentHeight, contract: contract, conn: conn, }, nil } Sia-1.0.4/modules/renter/proto/formcontract.go000066400000000000000000000230041302033070500213620ustar00rootroot00000000000000package proto import ( "errors" "net" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) const ( // estTxnSize is the estimated size of an encoded file contract // transaction set. estTxnSize = 2048 ) // FormContract forms a contract with a host and submits the contract // transaction to tpool. func FormContract(params ContractParams, txnBuilder transactionBuilder, tpool transactionPool) (modules.RenterContract, error) { // extract vars from params, for convenience host, filesize, startHeight, endHeight, refundAddress := params.Host, params.Filesize, params.StartHeight, params.EndHeight, params.RefundAddress // create our key ourSK, ourPK, err := crypto.GenerateKeyPair() if err != nil { return modules.RenterContract{}, err } ourPublicKey := types.SiaPublicKey{ Algorithm: types.SignatureEd25519, Key: ourPK[:], } // create unlock conditions uc := types.UnlockConditions{ PublicKeys: []types.SiaPublicKey{ourPublicKey, host.PublicKey}, SignaturesRequired: 2, } // calculate cost to renter and cost to host // TODO: clarify/abstract this math storageAllocation := host.StoragePrice.Mul64(filesize).Mul64(uint64(endHeight - startHeight)) hostCollateral := host.Collateral.Mul64(filesize).Mul64(uint64(endHeight - startHeight)) if hostCollateral.Cmp(host.MaxCollateral) > 0 { // TODO: if we have to cap the collateral, it probably means we shouldn't be using this host // (ok within a factor of 2) hostCollateral = host.MaxCollateral } hostPayout := hostCollateral.Add(host.ContractPrice) payout := storageAllocation.Add(hostPayout).Mul64(10406).Div64(10000) // renter pays for siafund fee renterCost := payout.Sub(hostCollateral) // check for negative currency if types.PostTax(startHeight, payout).Cmp(hostPayout) < 0 { return modules.RenterContract{}, errors.New("payout smaller than host payout") } // create file contract fc := types.FileContract{ FileSize: 0, FileMerkleRoot: crypto.Hash{}, // no proof possible without data WindowStart: endHeight, WindowEnd: endHeight + host.WindowSize, Payout: payout, UnlockHash: uc.UnlockHash(), RevisionNumber: 0, ValidProofOutputs: []types.SiacoinOutput{ // outputs need to account for tax {Value: types.PostTax(startHeight, payout).Sub(hostPayout), UnlockHash: refundAddress}, // collateral is returned to host {Value: hostPayout, UnlockHash: host.UnlockHash}, }, MissedProofOutputs: []types.SiacoinOutput{ // same as above {Value: types.PostTax(startHeight, payout).Sub(hostPayout), UnlockHash: refundAddress}, // same as above {Value: hostPayout, UnlockHash: host.UnlockHash}, // once we start doing revisions, we'll move some coins to the host and some to the void {Value: types.ZeroCurrency, UnlockHash: types.UnlockHash{}}, }, } // calculate transaction fee _, maxFee := tpool.FeeEstimation() fee := maxFee.Mul64(estTxnSize) // build transaction containing fc err = txnBuilder.FundSiacoins(renterCost.Add(fee)) if err != nil { return modules.RenterContract{}, err } txnBuilder.AddFileContract(fc) // add miner fee txnBuilder.AddMinerFee(fee) // create initial transaction set txn, parentTxns := txnBuilder.View() txnSet := append(parentTxns, txn) // initiate connection conn, err := net.DialTimeout("tcp", string(host.NetAddress), 15*time.Second) if err != nil { return modules.RenterContract{}, err } defer func() { _ = conn.Close() }() // allot time for sending RPC ID + verifySettings extendDeadline(conn, modules.NegotiateSettingsTime) if err = encoding.WriteObject(conn, modules.RPCFormContract); err != nil { return modules.RenterContract{}, err } // verify the host's settings and confirm its identity host, err = verifySettings(conn, host) if err != nil { return modules.RenterContract{}, err } if !host.AcceptingContracts { return modules.RenterContract{}, errors.New("host is not accepting contracts") } // allot time for negotiation extendDeadline(conn, modules.NegotiateFileContractTime) // send acceptance, txn signed by us, and pubkey if err = modules.WriteNegotiationAcceptance(conn); err != nil { return modules.RenterContract{}, errors.New("couldn't send initial acceptance: " + err.Error()) } if err = encoding.WriteObject(conn, txnSet); err != nil { return modules.RenterContract{}, errors.New("couldn't send the contract signed by us: " + err.Error()) } if err = encoding.WriteObject(conn, ourSK.PublicKey()); err != nil { return modules.RenterContract{}, errors.New("couldn't send our public key: " + err.Error()) } // read acceptance and txn signed by host if err = modules.ReadNegotiationAcceptance(conn); err != nil { return modules.RenterContract{}, errors.New("host did not accept our proposed contract: " + err.Error()) } // host now sends any new parent transactions, inputs and outputs that // were added to the transaction var newParents []types.Transaction var newInputs []types.SiacoinInput var newOutputs []types.SiacoinOutput if err = encoding.ReadObject(conn, &newParents, types.BlockSizeLimit); err != nil { return modules.RenterContract{}, errors.New("couldn't read the host's added parents: " + err.Error()) } if err = encoding.ReadObject(conn, &newInputs, types.BlockSizeLimit); err != nil { return modules.RenterContract{}, errors.New("couldn't read the host's added inputs: " + err.Error()) } if err = encoding.ReadObject(conn, &newOutputs, types.BlockSizeLimit); err != nil { return modules.RenterContract{}, errors.New("couldn't read the host's added outputs: " + err.Error()) } // merge txnAdditions with txnSet txnBuilder.AddParents(newParents) for _, input := range newInputs { txnBuilder.AddSiacoinInput(input) } for _, output := range newOutputs { txnBuilder.AddSiacoinOutput(output) } // sign the txn signedTxnSet, err := txnBuilder.Sign(true) if err != nil { return modules.RenterContract{}, modules.WriteNegotiationRejection(conn, errors.New("failed to sign transaction: "+err.Error())) } // calculate signatures added by the transaction builder var addedSignatures []types.TransactionSignature _, _, _, addedSignatureIndices := txnBuilder.ViewAdded() for _, i := range addedSignatureIndices { addedSignatures = append(addedSignatures, signedTxnSet[len(signedTxnSet)-1].TransactionSignatures[i]) } // create initial (no-op) revision, transaction, and signature initRevision := types.FileContractRevision{ ParentID: signedTxnSet[len(signedTxnSet)-1].FileContractID(0), UnlockConditions: uc, NewRevisionNumber: 1, NewFileSize: fc.FileSize, NewFileMerkleRoot: fc.FileMerkleRoot, NewWindowStart: fc.WindowStart, NewWindowEnd: fc.WindowEnd, NewValidProofOutputs: fc.ValidProofOutputs, NewMissedProofOutputs: fc.MissedProofOutputs, NewUnlockHash: fc.UnlockHash, } renterRevisionSig := types.TransactionSignature{ ParentID: crypto.Hash(initRevision.ParentID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{ FileContractRevisions: []uint64{0}, }, } revisionTxn := types.Transaction{ FileContractRevisions: []types.FileContractRevision{initRevision}, TransactionSignatures: []types.TransactionSignature{renterRevisionSig}, } encodedSig, err := crypto.SignHash(revisionTxn.SigHash(0), ourSK) if err != nil { return modules.RenterContract{}, modules.WriteNegotiationRejection(conn, errors.New("failed to sign revision transaction: "+err.Error())) } revisionTxn.TransactionSignatures[0].Signature = encodedSig[:] // Send acceptance and signatures if err = modules.WriteNegotiationAcceptance(conn); err != nil { return modules.RenterContract{}, errors.New("couldn't send transaction acceptance: " + err.Error()) } if err = encoding.WriteObject(conn, addedSignatures); err != nil { return modules.RenterContract{}, errors.New("couldn't send added signatures: " + err.Error()) } if err = encoding.WriteObject(conn, revisionTxn.TransactionSignatures[0]); err != nil { return modules.RenterContract{}, errors.New("couldn't send revision signature: " + err.Error()) } // Read the host acceptance and signatures. err = modules.ReadNegotiationAcceptance(conn) if err != nil { return modules.RenterContract{}, errors.New("host did not accept our signatures: " + err.Error()) } var hostSigs []types.TransactionSignature if err = encoding.ReadObject(conn, &hostSigs, 2e3); err != nil { return modules.RenterContract{}, errors.New("couldn't read the host's signatures: " + err.Error()) } for _, sig := range hostSigs { txnBuilder.AddTransactionSignature(sig) } var hostRevisionSig types.TransactionSignature if err = encoding.ReadObject(conn, &hostRevisionSig, 2e3); err != nil { return modules.RenterContract{}, errors.New("couldn't read the host's revision signature: " + err.Error()) } revisionTxn.TransactionSignatures = append(revisionTxn.TransactionSignatures, hostRevisionSig) // Construct the final transaction. txn, parentTxns = txnBuilder.View() txnSet = append(parentTxns, txn) // Submit to blockchain. err = tpool.AcceptTransactionSet(txnSet) if err == modules.ErrDuplicateTransactionSet { // as long as it made it into the transaction pool, we're good err = nil } if err != nil { return modules.RenterContract{}, err } // calculate contract ID fcid := txn.FileContractID(0) return modules.RenterContract{ FileContract: fc, ID: fcid, LastRevision: initRevision, LastRevisionTxn: revisionTxn, NetAddress: host.NetAddress, SecretKey: ourSK, }, nil } Sia-1.0.4/modules/renter/proto/negotiate.go000066400000000000000000000235771302033070500206570ustar00rootroot00000000000000package proto import ( "errors" "net" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // extendDeadline is a helper function for extending the connection timeout. func extendDeadline(conn net.Conn, d time.Duration) { _ = conn.SetDeadline(time.Now().Add(d)) } // startRevision is run at the beginning of each revision iteration. It reads // the host's settings confirms that the values are acceptable, and writes an acceptance. func startRevision(conn net.Conn, host modules.HostDBEntry) error { // verify the host's settings and confirm its identity _, err := verifySettings(conn, host) if err != nil { return err } return modules.WriteNegotiationAcceptance(conn) } // startDownload is run at the beginning of each download iteration. It reads // the host's settings confirms that the values are acceptable, and writes an acceptance. func startDownload(conn net.Conn, host modules.HostDBEntry) error { // verify the host's settings and confirm its identity _, err := verifySettings(conn, host) if err != nil { return err } return modules.WriteNegotiationAcceptance(conn) } // verifySettings reads a signed HostSettings object from conn, validates the // signature, and checks for discrepancies between the known settings and the // received settings. If there is a discrepancy, the hostDB is notified. The // received settings are returned. func verifySettings(conn net.Conn, host modules.HostDBEntry) (modules.HostDBEntry, error) { // convert host key (types.SiaPublicKey) to a crypto.PublicKey if host.PublicKey.Algorithm != types.SignatureEd25519 || len(host.PublicKey.Key) != crypto.PublicKeySize { build.Critical("hostdb did not filter out host with wrong signature algorithm:", host.PublicKey.Algorithm) return modules.HostDBEntry{}, errors.New("host used unsupported signature algorithm") } var pk crypto.PublicKey copy(pk[:], host.PublicKey.Key) // read signed host settings var recvSettings modules.HostExternalSettings if err := crypto.ReadSignedObject(conn, &recvSettings, modules.NegotiateMaxHostExternalSettingsLen, pk); err != nil { return modules.HostDBEntry{}, errors.New("couldn't read host's settings: " + err.Error()) } // TODO: check recvSettings against host.HostExternalSettings. If there is // a discrepancy, write the error to conn. if recvSettings.NetAddress != host.NetAddress { // for now, just overwrite the NetAddress, since we know that // host.NetAddress works (it was the one we dialed to get conn) recvSettings.NetAddress = host.NetAddress } host.HostExternalSettings = recvSettings return host, nil } // verifyRecentRevision confirms that the host and contractor agree upon the current // state of the contract being revised. func verifyRecentRevision(conn net.Conn, contract modules.RenterContract) error { // send contract ID if err := encoding.WriteObject(conn, contract.ID); err != nil { return errors.New("couldn't send contract ID: " + err.Error()) } // read challenge var challenge crypto.Hash if err := encoding.ReadObject(conn, &challenge, 32); err != nil { return errors.New("couldn't read challenge: " + err.Error()) } // sign and return sig, err := crypto.SignHash(challenge, contract.SecretKey) if err != nil { return err } else if err := encoding.WriteObject(conn, sig); err != nil { return errors.New("couldn't send challenge response: " + err.Error()) } // read acceptance if err := modules.ReadNegotiationAcceptance(conn); err != nil { return errors.New("host did not accept revision request: " + err.Error()) } // read last revision and signatures var lastRevision types.FileContractRevision var hostSignatures []types.TransactionSignature if err := encoding.ReadObject(conn, &lastRevision, 2048); err != nil { return errors.New("couldn't read last revision: " + err.Error()) } if err := encoding.ReadObject(conn, &hostSignatures, 2048); err != nil { return errors.New("couldn't read host signatures: " + err.Error()) } // Check that the unlock hashes match; if they do not, something is // seriously wrong. Otherwise, check that the revision numbers match. if lastRevision.UnlockConditions.UnlockHash() != contract.LastRevision.UnlockConditions.UnlockHash() { return errors.New("unlock conditions do not match") } else if lastRevision.NewRevisionNumber != contract.LastRevision.NewRevisionNumber { return &recentRevisionError{contract.LastRevision.NewRevisionNumber, lastRevision.NewRevisionNumber} } // NOTE: we can fake the blockheight here because it doesn't affect // verification; it just needs to be above the fork height and below the // contract expiration (which was checked earlier). return modules.VerifyFileContractRevisionTransactionSignatures(lastRevision, hostSignatures, contract.FileContract.WindowStart-1) } // negotiateRevision sends a revision and actions to the host for approval, // completing one iteration of the revision loop. func negotiateRevision(conn net.Conn, rev types.FileContractRevision, secretKey crypto.SecretKey) (types.Transaction, error) { // create transaction containing the revision signedTxn := types.Transaction{ FileContractRevisions: []types.FileContractRevision{rev}, TransactionSignatures: []types.TransactionSignature{{ ParentID: crypto.Hash(rev.ParentID), CoveredFields: types.CoveredFields{FileContractRevisions: []uint64{0}}, PublicKeyIndex: 0, // renter key is always first -- see formContract }}, } // sign the transaction encodedSig, _ := crypto.SignHash(signedTxn.SigHash(0), secretKey) // no error possible signedTxn.TransactionSignatures[0].Signature = encodedSig[:] // send the revision if err := encoding.WriteObject(conn, rev); err != nil { return types.Transaction{}, errors.New("couldn't send revision: " + err.Error()) } // read acceptance if err := modules.ReadNegotiationAcceptance(conn); err != nil { return types.Transaction{}, errors.New("host did not accept revision: " + err.Error()) } // send the new transaction signature if err := encoding.WriteObject(conn, signedTxn.TransactionSignatures[0]); err != nil { return types.Transaction{}, errors.New("couldn't send transaction signature: " + err.Error()) } // read the host's acceptance and transaction signature // NOTE: if the host sends ErrStopResponse, we should continue processing // the revision, but return the error anyway. responseErr := modules.ReadNegotiationAcceptance(conn) if responseErr != nil && responseErr != modules.ErrStopResponse { return types.Transaction{}, errors.New("host did not accept transaction signature: " + responseErr.Error()) } var hostSig types.TransactionSignature if err := encoding.ReadObject(conn, &hostSig, 16e3); err != nil { return types.Transaction{}, errors.New("couldn't read host's signature: " + err.Error()) } // add the signature to the transaction and verify it // NOTE: we can fake the blockheight here because it doesn't affect // verification; it just needs to be above the fork height and below the // contract expiration (which was checked earlier). verificationHeight := rev.NewWindowStart - 1 signedTxn.TransactionSignatures = append(signedTxn.TransactionSignatures, hostSig) if err := signedTxn.StandaloneValid(verificationHeight); err != nil { return types.Transaction{}, err } // if the host sent ErrStopResponse, return it return signedTxn, responseErr } // newRevision creates a copy of current with its revision number incremented, // and with cost transferred from the renter to the host. func newRevision(current types.FileContractRevision, cost types.Currency) types.FileContractRevision { rev := current // need to manually copy slice memory rev.NewValidProofOutputs = make([]types.SiacoinOutput, 2) rev.NewMissedProofOutputs = make([]types.SiacoinOutput, 3) copy(rev.NewValidProofOutputs, current.NewValidProofOutputs) copy(rev.NewMissedProofOutputs, current.NewMissedProofOutputs) // move valid payout from renter to host rev.NewValidProofOutputs[0].Value = current.NewValidProofOutputs[0].Value.Sub(cost) rev.NewValidProofOutputs[1].Value = current.NewValidProofOutputs[1].Value.Add(cost) // move missed payout from renter to void rev.NewMissedProofOutputs[0].Value = current.NewMissedProofOutputs[0].Value.Sub(cost) rev.NewMissedProofOutputs[2].Value = current.NewMissedProofOutputs[2].Value.Add(cost) // increment revision number rev.NewRevisionNumber++ return rev } // newDownloadRevision revises the current revision to cover the cost of // downloading data. func newDownloadRevision(current types.FileContractRevision, downloadCost types.Currency) types.FileContractRevision { return newRevision(current, downloadCost) } // newUploadRevision revises the current revision to cover the cost of // uploading a sector. func newUploadRevision(current types.FileContractRevision, merkleRoot crypto.Hash, price, collateral types.Currency) types.FileContractRevision { rev := newRevision(current, price) // move collateral from host to void rev.NewMissedProofOutputs[1].Value = rev.NewMissedProofOutputs[1].Value.Sub(collateral) rev.NewMissedProofOutputs[2].Value = rev.NewMissedProofOutputs[2].Value.Add(collateral) // set new filesize and Merkle root rev.NewFileSize += modules.SectorSize rev.NewFileMerkleRoot = merkleRoot return rev } // newDeleteRevision revises the current revision to cover the cost of // deleting a sector. func newDeleteRevision(current types.FileContractRevision, merkleRoot crypto.Hash) types.FileContractRevision { rev := newRevision(current, types.ZeroCurrency) rev.NewFileSize -= modules.SectorSize rev.NewFileMerkleRoot = merkleRoot return rev } // newModifyRevision revises the current revision to cover the cost of // modifying a sector. func newModifyRevision(current types.FileContractRevision, merkleRoot crypto.Hash, uploadCost types.Currency) types.FileContractRevision { rev := newRevision(current, uploadCost) rev.NewFileMerkleRoot = merkleRoot return rev } Sia-1.0.4/modules/renter/proto/negotiate_test.go000066400000000000000000000043561302033070500217100ustar00rootroot00000000000000package proto import ( "errors" "net" "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // TestNegotiateRevisionStopResponse tests that when the host sends // StopResponse, the renter continues processing the revision instead of // immediately terminating. func TestNegotiateRevisionStopResponse(t *testing.T) { // simulate a renter-host connection rConn, hConn := net.Pipe() // handle the host's half of the pipe go func() { defer hConn.Close() // read revision encoding.ReadObject(hConn, new(types.FileContractRevision), 1<<22) // write acceptance modules.WriteNegotiationAcceptance(hConn) // read txn signature encoding.ReadObject(hConn, new(types.TransactionSignature), 1<<22) // write StopResponse modules.WriteNegotiationStop(hConn) // write txn signature encoding.WriteObject(hConn, types.TransactionSignature{}) }() // since the host wrote StopResponse, we should proceed to validating the // transaction. This will return a known error because we are supplying an // empty revision. _, err := negotiateRevision(rConn, types.FileContractRevision{}, crypto.SecretKey{}) if err != types.ErrFileContractWindowStartViolation { t.Fatalf("expected %q, got \"%v\"", types.ErrFileContractWindowStartViolation, err) } rConn.Close() // same as above, but send an error instead of StopResponse. The error // should be returned by negotiateRevision immediately (if it is not, we // should expect to see a transaction validation error instead). rConn, hConn = net.Pipe() go func() { defer hConn.Close() encoding.ReadObject(hConn, new(types.FileContractRevision), 1<<22) modules.WriteNegotiationAcceptance(hConn) encoding.ReadObject(hConn, new(types.TransactionSignature), 1<<22) // write a sentinel error modules.WriteNegotiationRejection(hConn, errors.New("sentinel")) encoding.WriteObject(hConn, types.TransactionSignature{}) }() expectedErr := "host did not accept transaction signature: sentinel" _, err = negotiateRevision(rConn, types.FileContractRevision{}, crypto.SecretKey{}) if err == nil || err.Error() != expectedErr { t.Fatalf("expected %q, got \"%v\"", expectedErr, err) } rConn.Close() } Sia-1.0.4/modules/renter/proto/proto.go000066400000000000000000000035001302033070500200230ustar00rootroot00000000000000package proto import ( "fmt" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // dependencies type ( transactionBuilder interface { AddFileContract(types.FileContract) uint64 AddMinerFee(types.Currency) uint64 AddParents([]types.Transaction) AddSiacoinInput(types.SiacoinInput) uint64 AddSiacoinOutput(types.SiacoinOutput) uint64 AddTransactionSignature(types.TransactionSignature) uint64 FundSiacoins(types.Currency) error Sign(bool) ([]types.Transaction, error) View() (types.Transaction, []types.Transaction) ViewAdded() (parents, coins, funds, signatures []int) } transactionPool interface { AcceptTransactionSet([]types.Transaction) error FeeEstimation() (min types.Currency, max types.Currency) } ) // ContractParams are supplied as an argument to FormContract. type ContractParams struct { Host modules.HostDBEntry Filesize uint64 StartHeight types.BlockHeight EndHeight types.BlockHeight RefundAddress types.UnlockHash // TODO: add optional keypair } // A revisionSaver is called just before we send our revision signature to the host; this // allows the revision and Merkle roots to be reloaded later if we desync from the host. type revisionSaver func(types.FileContractRevision, []crypto.Hash) error // A recentRevisionError occurs if the host reports a different revision // number than expected. type recentRevisionError struct { ours, theirs uint64 } func (e *recentRevisionError) Error() string { return fmt.Sprintf("our revision number (%v) does not match the host's (%v)", e.ours, e.theirs) } // IsRevisionMismatch returns true if err was caused by the host reporting a // different revision number than expected. func IsRevisionMismatch(err error) bool { _, ok := err.(*recentRevisionError) return ok } Sia-1.0.4/modules/renter/proto/renew.go000066400000000000000000000246071302033070500200130ustar00rootroot00000000000000package proto import ( "errors" "net" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // Renew negotiates a new contract for data already stored with a host, and // submits the new contract transaction to tpool. func Renew(contract modules.RenterContract, params ContractParams, txnBuilder transactionBuilder, tpool transactionPool) (modules.RenterContract, error) { // extract vars from params, for convenience host, filesize, startHeight, endHeight, refundAddress := params.Host, params.Filesize, params.StartHeight, params.EndHeight, params.RefundAddress ourSK := contract.SecretKey // calculate cost to renter and cost to host storageAllocation := host.StoragePrice.Mul64(filesize).Mul64(uint64(endHeight - startHeight)) hostCollateral := host.Collateral.Mul64(filesize).Mul64(uint64(endHeight - startHeight)) if hostCollateral.Cmp(host.MaxCollateral) > 0 { // TODO: if we have to cap the collateral, it probably means we shouldn't be using this host // (ok within a factor of 2) hostCollateral = host.MaxCollateral } // Calculate additional basePrice and baseCollateral. If the contract // height did not increase, basePrice and baseCollateral are zero. var basePrice, baseCollateral types.Currency if endHeight+host.WindowSize > contract.LastRevision.NewWindowEnd { timeExtension := uint64((endHeight + host.WindowSize) - contract.LastRevision.NewWindowEnd) basePrice = host.StoragePrice.Mul64(contract.LastRevision.NewFileSize).Mul64(timeExtension) // cost of data already covered by contract, i.e. lastrevision.Filesize baseCollateral = host.Collateral.Mul64(contract.LastRevision.NewFileSize).Mul64(timeExtension) // same but collateral } hostPayout := hostCollateral.Add(host.ContractPrice).Add(basePrice) payout := storageAllocation.Add(hostCollateral.Add(host.ContractPrice)).Mul64(10406).Div64(10000) // renter covers siafund fee renterCost := payout.Sub(hostCollateral) // check for negative currency if types.PostTax(startHeight, payout).Cmp(hostPayout) < 0 { return modules.RenterContract{}, errors.New("payout smaller than host payout") } else if hostCollateral.Cmp(baseCollateral) < 0 { return modules.RenterContract{}, errors.New("new collateral smaller than old collateral") } // create file contract fc := types.FileContract{ FileSize: contract.LastRevision.NewFileSize, FileMerkleRoot: contract.LastRevision.NewFileMerkleRoot, WindowStart: endHeight, WindowEnd: endHeight + host.WindowSize, Payout: payout, UnlockHash: contract.LastRevision.NewUnlockHash, RevisionNumber: 0, ValidProofOutputs: []types.SiacoinOutput{ // renter {Value: types.PostTax(startHeight, payout).Sub(hostPayout), UnlockHash: refundAddress}, // host {Value: hostPayout, UnlockHash: host.UnlockHash}, }, MissedProofOutputs: []types.SiacoinOutput{ // renter {Value: types.PostTax(startHeight, payout).Sub(hostPayout), UnlockHash: refundAddress}, // host gets its unused collateral back, plus the contract price {Value: hostCollateral.Sub(baseCollateral).Add(host.ContractPrice), UnlockHash: host.UnlockHash}, // void gets the spent storage fees, plus the collateral being risked {Value: basePrice.Add(baseCollateral), UnlockHash: types.UnlockHash{}}, }, } // calculate transaction fee _, maxFee := tpool.FeeEstimation() fee := maxFee.Mul64(estTxnSize) // build transaction containing fc err := txnBuilder.FundSiacoins(renterCost.Add(fee)) if err != nil { return modules.RenterContract{}, err } txnBuilder.AddFileContract(fc) // add miner fee txnBuilder.AddMinerFee(fee) // create initial transaction set txn, parentTxns := txnBuilder.View() txnSet := append(parentTxns, txn) // initiate connection conn, err := net.DialTimeout("tcp", string(host.NetAddress), 15*time.Second) if err != nil { return modules.RenterContract{}, err } defer func() { _ = conn.Close() }() // allot time for sending RPC ID, verifyRecentRevision, and verifySettings extendDeadline(conn, modules.NegotiateRecentRevisionTime+modules.NegotiateSettingsTime) if err = encoding.WriteObject(conn, modules.RPCRenewContract); err != nil { return modules.RenterContract{}, errors.New("couldn't initiate RPC: " + err.Error()) } // verify that both parties are renewing the same contract if err = verifyRecentRevision(conn, contract); err != nil { return modules.RenterContract{}, errors.New("revision exchange failed: " + err.Error()) } // verify the host's settings and confirm its identity host, err = verifySettings(conn, host) if err != nil { return modules.RenterContract{}, errors.New("settings exchange failed: " + err.Error()) } if !host.AcceptingContracts { return modules.RenterContract{}, errors.New("host is not accepting contracts") } // allot time for negotiation extendDeadline(conn, modules.NegotiateRenewContractTime) // send acceptance, txn signed by us, and pubkey if err = modules.WriteNegotiationAcceptance(conn); err != nil { return modules.RenterContract{}, errors.New("couldn't send initial acceptance: " + err.Error()) } if err = encoding.WriteObject(conn, txnSet); err != nil { return modules.RenterContract{}, errors.New("couldn't send the contract signed by us: " + err.Error()) } if err = encoding.WriteObject(conn, ourSK.PublicKey()); err != nil { return modules.RenterContract{}, errors.New("couldn't send our public key: " + err.Error()) } // read acceptance and txn signed by host if err = modules.ReadNegotiationAcceptance(conn); err != nil { return modules.RenterContract{}, errors.New("host did not accept our proposed contract: " + err.Error()) } // host now sends any new parent transactions, inputs and outputs that // were added to the transaction var newParents []types.Transaction var newInputs []types.SiacoinInput var newOutputs []types.SiacoinOutput if err = encoding.ReadObject(conn, &newParents, types.BlockSizeLimit); err != nil { return modules.RenterContract{}, errors.New("couldn't read the host's added parents: " + err.Error()) } if err = encoding.ReadObject(conn, &newInputs, types.BlockSizeLimit); err != nil { return modules.RenterContract{}, errors.New("couldn't read the host's added inputs: " + err.Error()) } if err = encoding.ReadObject(conn, &newOutputs, types.BlockSizeLimit); err != nil { return modules.RenterContract{}, errors.New("couldn't read the host's added outputs: " + err.Error()) } // merge txnAdditions with txnSet txnBuilder.AddParents(newParents) for _, input := range newInputs { txnBuilder.AddSiacoinInput(input) } for _, output := range newOutputs { txnBuilder.AddSiacoinOutput(output) } // sign the txn signedTxnSet, err := txnBuilder.Sign(true) if err != nil { return modules.RenterContract{}, modules.WriteNegotiationRejection(conn, errors.New("failed to sign transaction: "+err.Error())) } // calculate signatures added by the transaction builder var addedSignatures []types.TransactionSignature _, _, _, addedSignatureIndices := txnBuilder.ViewAdded() for _, i := range addedSignatureIndices { addedSignatures = append(addedSignatures, signedTxnSet[len(signedTxnSet)-1].TransactionSignatures[i]) } // create initial (no-op) revision, transaction, and signature initRevision := types.FileContractRevision{ ParentID: signedTxnSet[len(signedTxnSet)-1].FileContractID(0), UnlockConditions: contract.LastRevision.UnlockConditions, NewRevisionNumber: 1, NewFileSize: fc.FileSize, NewFileMerkleRoot: fc.FileMerkleRoot, NewWindowStart: fc.WindowStart, NewWindowEnd: fc.WindowEnd, NewValidProofOutputs: fc.ValidProofOutputs, NewMissedProofOutputs: fc.MissedProofOutputs, NewUnlockHash: fc.UnlockHash, } renterRevisionSig := types.TransactionSignature{ ParentID: crypto.Hash(initRevision.ParentID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{ FileContractRevisions: []uint64{0}, }, } revisionTxn := types.Transaction{ FileContractRevisions: []types.FileContractRevision{initRevision}, TransactionSignatures: []types.TransactionSignature{renterRevisionSig}, } encodedSig, err := crypto.SignHash(revisionTxn.SigHash(0), ourSK) if err != nil { return modules.RenterContract{}, modules.WriteNegotiationRejection(conn, errors.New("failed to sign revision transaction: "+err.Error())) } revisionTxn.TransactionSignatures[0].Signature = encodedSig[:] // Send acceptance and signatures if err = modules.WriteNegotiationAcceptance(conn); err != nil { return modules.RenterContract{}, errors.New("couldn't send transaction acceptance: " + err.Error()) } if err = encoding.WriteObject(conn, addedSignatures); err != nil { return modules.RenterContract{}, errors.New("couldn't send added signatures: " + err.Error()) } if err = encoding.WriteObject(conn, revisionTxn.TransactionSignatures[0]); err != nil { return modules.RenterContract{}, errors.New("couldn't send revision signature: " + err.Error()) } // Read the host acceptance and signatures. err = modules.ReadNegotiationAcceptance(conn) if err != nil { return modules.RenterContract{}, errors.New("host did not accept our signatures: " + err.Error()) } var hostSigs []types.TransactionSignature if err = encoding.ReadObject(conn, &hostSigs, 2e3); err != nil { return modules.RenterContract{}, errors.New("couldn't read the host's signatures: " + err.Error()) } for _, sig := range hostSigs { txnBuilder.AddTransactionSignature(sig) } var hostRevisionSig types.TransactionSignature if err = encoding.ReadObject(conn, &hostRevisionSig, 2e3); err != nil { return modules.RenterContract{}, errors.New("couldn't read the host's revision signature: " + err.Error()) } revisionTxn.TransactionSignatures = append(revisionTxn.TransactionSignatures, hostRevisionSig) // Construct the final transaction. txn, parentTxns = txnBuilder.View() txnSet = append(parentTxns, txn) // Submit to blockchain. err = tpool.AcceptTransactionSet(txnSet) if err == modules.ErrDuplicateTransactionSet { // as long as it made it into the transaction pool, we're good err = nil } if err != nil { return modules.RenterContract{}, err } // calculate contract ID fcid := txn.FileContractID(0) return modules.RenterContract{ FileContract: fc, ID: fcid, LastRevision: initRevision, LastRevisionTxn: revisionTxn, MerkleRoots: contract.MerkleRoots, NetAddress: host.NetAddress, SecretKey: ourSK, }, nil } Sia-1.0.4/modules/renter/renter.go000066400000000000000000000123021302033070500170140ustar00rootroot00000000000000package renter import ( "errors" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/renter/contractor" "github.com/NebulousLabs/Sia/modules/renter/hostdb" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/sync" "github.com/NebulousLabs/Sia/types" ) var ( errNilCS = errors.New("cannot create renter with nil consensus set") errNilTpool = errors.New("cannot create renter with nil transaction pool") errNilHdb = errors.New("cannot create renter with nil hostdb") ) // A hostDB is a database of hosts that the renter can use for figuring out who // to upload to, and download from. type hostDB interface { // ActiveHosts returns the list of hosts that are actively being selected // from. ActiveHosts() []modules.HostDBEntry // AllHosts returns the full list of hosts known to the hostdb, sorted in // order of preference. AllHosts() []modules.HostDBEntry // AverageContractPrice returns the average contract price of a host. AverageContractPrice() types.Currency // Close closes the hostdb. Close() error // IsOffline reports whether a host is consider offline. IsOffline(modules.NetAddress) bool } // A hostContractor negotiates, revises, renews, and provides access to file // contracts. type hostContractor interface { // SetAllowance sets the amount of money the contractor is allowed to // spend on contracts over a given time period, divided among the number // of hosts specified. Note that contractor can start forming contracts as // soon as SetAllowance is called; that is, it may block. SetAllowance(modules.Allowance) error // Allowance returns the current allowance Allowance() modules.Allowance // Contract returns the latest contract formed with the specified host. Contract(modules.NetAddress) (modules.RenterContract, bool) // Contracts returns the contracts formed by the contractor. Contracts() []modules.RenterContract // Editor creates an Editor from the specified contract ID, allowing the // insertion, deletion, and modification of sectors. Editor(types.FileContractID) (contractor.Editor, error) // FinancialMetrics returns the financial metrics of the contractor. FinancialMetrics() modules.RenterFinancialMetrics // Downloader creates a Downloader from the specified contract ID, // allowing the retrieval of sectors. Downloader(types.FileContractID) (contractor.Downloader, error) } // A trackedFile contains metadata about files being tracked by the Renter. // Tracked files are actively repaired by the Renter. By default, files // uploaded by the user are tracked, and files that are added (via loading a // .sia file) are not. type trackedFile struct { // location of original file on disk RepairPath string } // A Renter is responsible for tracking all of the files that a user has // uploaded to Sia, as well as the locations and health of these files. type Renter struct { // modules cs modules.ConsensusSet // resources hostDB hostDB hostContractor hostContractor log *persist.Logger // variables files map[string]*file tracking map[string]trackedFile // map from nickname to metadata downloadQueue []*download uploading bool downloading bool // constants persistDir string mu *sync.RWMutex } // New returns an initialized renter. func New(cs modules.ConsensusSet, wallet modules.Wallet, tpool modules.TransactionPool, persistDir string) (*Renter, error) { hdb, err := hostdb.New(cs, persistDir) if err != nil { return nil, err } hc, err := contractor.New(cs, wallet, tpool, hdb, persistDir) if err != nil { return nil, err } return newRenter(cs, tpool, hdb, hc, persistDir) } // newRenter initializes a renter and returns it. func newRenter(cs modules.ConsensusSet, tpool modules.TransactionPool, hdb hostDB, hc hostContractor, persistDir string) (*Renter, error) { if cs == nil { return nil, errNilCS } if tpool == nil { return nil, errNilTpool } if hdb == nil { // Nil hdb currently allowed for testing purposes. :( // return nil, errNilHdb } r := &Renter{ cs: cs, hostDB: hdb, hostContractor: hc, files: make(map[string]*file), tracking: make(map[string]trackedFile), persistDir: persistDir, mu: sync.New(modules.SafeMutexDelay, 1), } if err := r.initPersist(); err != nil { return nil, err } go r.threadedRepairLoop() return r, nil } // Close closes the Renter and its dependencies func (r *Renter) Close() error { return r.hostDB.Close() } // hostdb passthroughs func (r *Renter) ActiveHosts() []modules.HostDBEntry { return r.hostDB.ActiveHosts() } func (r *Renter) AllHosts() []modules.HostDBEntry { return r.hostDB.AllHosts() } // contractor passthroughs func (r *Renter) Contracts() []modules.RenterContract { return r.hostContractor.Contracts() } func (r *Renter) FinancialMetrics() modules.RenterFinancialMetrics { return r.hostContractor.FinancialMetrics() } func (r *Renter) Settings() modules.RenterSettings { return modules.RenterSettings{ Allowance: r.hostContractor.Allowance(), } } func (r *Renter) SetSettings(s modules.RenterSettings) error { return r.hostContractor.SetAllowance(s.Allowance) } // enforce that Renter satisfies the modules.Renter interface var _ modules.Renter = (*Renter)(nil) Sia-1.0.4/modules/renter/renter_test.go000066400000000000000000000123631302033070500200620ustar00rootroot00000000000000package renter import ( "path/filepath" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/consensus" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/modules/miner" "github.com/NebulousLabs/Sia/modules/renter/contractor" "github.com/NebulousLabs/Sia/modules/transactionpool" "github.com/NebulousLabs/Sia/modules/wallet" "github.com/NebulousLabs/Sia/types" ) // renterTester contains all of the modules that are used while testing the renter. type renterTester struct { cs modules.ConsensusSet gateway modules.Gateway miner modules.TestMiner tpool modules.TransactionPool wallet modules.Wallet walletKey crypto.TwofishKey renter *Renter } // Close shuts down the renter tester. func (rt *renterTester) Close() error { rt.wallet.Lock() rt.cs.Close() rt.gateway.Close() return nil } // newRenterTester creates a ready-to-use renter tester with money in the // wallet. func newRenterTester(name string) (*renterTester, error) { // Create the modules. testdir := build.TempDir("renter", name) g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, err } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { return nil, err } w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } key, err := crypto.GenerateTwofishKey() if err != nil { return nil, err } _, err = w.Encrypt(key) if err != nil { return nil, err } err = w.Unlock(key) if err != nil { return nil, err } r, err := New(cs, w, tp, filepath.Join(testdir, modules.RenterDir)) if err != nil { return nil, err } m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.MinerDir)) if err != nil { return nil, err } // Assemble all pieces into a renter tester. rt := &renterTester{ cs: cs, gateway: g, miner: m, tpool: tp, wallet: w, renter: r, } // Mine blocks until there is money in the wallet. for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ { _, err := rt.miner.AddBlock() if err != nil { return nil, err } } return rt, nil } // newContractorTester creates a renterTester, but with the supplied // hostContractor. func newContractorTester(name string, hdb hostDB, hc hostContractor) (*renterTester, error) { // Create the modules. testdir := build.TempDir("renter", name) g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, err } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { return nil, err } w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } key, err := crypto.GenerateTwofishKey() if err != nil { return nil, err } _, err = w.Encrypt(key) if err != nil { return nil, err } err = w.Unlock(key) if err != nil { return nil, err } r, err := newRenter(cs, tp, hdb, hc, filepath.Join(testdir, modules.RenterDir)) if err != nil { return nil, err } m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.MinerDir)) if err != nil { return nil, err } // Assemble all pieces into a renter tester. rt := &renterTester{ cs: cs, gateway: g, miner: m, tpool: tp, wallet: w, renter: r, } // Mine blocks until there is money in the wallet. for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ { _, err := rt.miner.AddBlock() if err != nil { return nil, err } } return rt, nil } // stubHostDB is the minimal implementation of the hostDB interface. It can be // embedded in other mock hostDB types, removing the need to reimplement all // of the hostDB's methods on every mock. type stubHostDB struct{} func (stubHostDB) ActiveHosts() []modules.HostDBEntry { return nil } func (stubHostDB) AllHosts() []modules.HostDBEntry { return nil } func (stubHostDB) AverageContractPrice() types.Currency { return types.Currency{} } func (stubHostDB) Close() error { return nil } func (stubHostDB) IsOffline(modules.NetAddress) bool { return true } // stubContractor is the minimal implementation of the hostContractor // interface. type stubContractor struct{} func (stubContractor) SetAllowance(modules.Allowance) error { return nil } func (stubContractor) Allowance() modules.Allowance { return modules.Allowance{} } func (stubContractor) Contract(modules.NetAddress) (modules.RenterContract, bool) { return modules.RenterContract{}, false } func (stubContractor) Contracts() []modules.RenterContract { return nil } func (stubContractor) FinancialMetrics() (m modules.RenterFinancialMetrics) { return } func (stubContractor) Editor(types.FileContractID) (contractor.Editor, error) { return nil, nil } func (stubContractor) Downloader(types.FileContractID) (contractor.Downloader, error) { return nil, nil } Sia-1.0.4/modules/renter/repair.go000066400000000000000000000207771302033070500170160ustar00rootroot00000000000000package renter import ( "fmt" "io" "os" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/renter/contractor" "github.com/NebulousLabs/Sia/types" ) const ( // repairThreads is the number of repairs that can run concurrently. repairThreads = 10 ) // When a file contract is within 'renewThreshold' blocks of expiring, the renter // will attempt to renew the contract. var renewThreshold = func() types.BlockHeight { switch build.Release { case "testing": return 10 case "dev": return 200 default: return 144 * 7 * 3 // 3 weeks - to soon be 6 weeks. } }() // hostErr and hostErrs are helpers for reporting repair errors. The actual // Error implementations aren't that important; we just need to be able to // extract the NetAddress of the failed host. type hostErr struct { host modules.NetAddress err error } func (he hostErr) Error() string { return fmt.Sprintf("host %v failed: %v", he.host, he.err) } type hostErrs []*hostErr func (hs hostErrs) Error() string { var errs []error for _, h := range hs { errs = append(errs, h) } return build.JoinErrors(errs, "\n").Error() } // repair attempts to repair a file chunk by uploading its pieces to more // hosts. func (f *file) repair(chunkIndex uint64, missingPieces []uint64, r io.ReaderAt, hosts []contractor.Editor) error { // read chunk data and encode chunk := make([]byte, f.chunkSize()) _, err := r.ReadAt(chunk, int64(chunkIndex*f.chunkSize())) if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { return err } pieces, err := f.erasureCode.Encode(chunk) if err != nil { return err } // encrypt pieces for i := range pieces { key := deriveKey(f.masterKey, chunkIndex, uint64(i)) pieces[i], err = key.EncryptBytes(pieces[i]) if err != nil { return err } } // upload one piece per host numPieces := len(missingPieces) if len(hosts) < numPieces { numPieces = len(hosts) } errChan := make(chan *hostErr) for i := 0; i < numPieces; i++ { go func(pieceIndex uint64, host contractor.Editor) { // upload data to host root, err := host.Upload(pieces[pieceIndex]) if err != nil { errChan <- &hostErr{host.Address(), err} return } // create contract entry, if necessary f.mu.Lock() contract, ok := f.contracts[host.ContractID()] if !ok { contract = fileContract{ ID: host.ContractID(), IP: host.Address(), WindowStart: host.EndHeight(), } } // update contract contract.Pieces = append(contract.Pieces, pieceData{ Chunk: chunkIndex, Piece: pieceIndex, MerkleRoot: root, }) f.contracts[host.ContractID()] = contract f.mu.Unlock() errChan <- nil }(missingPieces[i], hosts[i]) } var errs hostErrs for i := 0; i < numPieces; i++ { err := <-errChan if err != nil { errs = append(errs, err) } } if errs != nil { return errs } return nil } // incompleteChunks returns a map of chunks containing pieces that have not // been uploaded. func (f *file) incompleteChunks() map[uint64][]uint64 { f.mu.RLock() defer f.mu.RUnlock() present := make([][]bool, f.numChunks()) for i := range present { present[i] = make([]bool, f.erasureCode.NumPieces()) } for _, fc := range f.contracts { for _, p := range fc.Pieces { present[p.Chunk][p.Piece] = true } } incomplete := make(map[uint64][]uint64) for chunkIndex, pieceBools := range present { for pieceIndex, ok := range pieceBools { if !ok { incomplete[uint64(chunkIndex)] = append(incomplete[uint64(chunkIndex)], uint64(pieceIndex)) } } } return incomplete } // chunkHosts returns the hosts storing the given chunk. func (f *file) chunkHosts(chunk uint64) []modules.NetAddress { f.mu.RLock() defer f.mu.RUnlock() var old []modules.NetAddress for _, fc := range f.contracts { for _, p := range fc.Pieces { if p.Chunk == chunk { old = append(old, fc.IP) break } } } return old } // expiringContracts returns the contracts that will expire soon. // TODO: what if contract has fully expired? func (f *file) expiringContracts(height types.BlockHeight) []fileContract { f.mu.RLock() defer f.mu.RUnlock() var expiring []fileContract for _, fc := range f.contracts { if height >= fc.WindowStart-renewThreshold { expiring = append(expiring, fc) } } return expiring } // offlineChunks returns the chunks belonging to "offline" hosts -- hosts that // do not meet uptime requirements. Importantly, only chunks missing more than // half their redundancy are returned. func (f *file) offlineChunks(hdb hostDB) map[uint64][]uint64 { f.mu.RLock() defer f.mu.RUnlock() // mark all pieces belonging to offline hosts. offline := make(map[uint64][]uint64) for _, fc := range f.contracts { if hdb.IsOffline(fc.IP) { for _, p := range fc.Pieces { offline[p.Chunk] = append(offline[p.Chunk], p.Piece) } } } // filter out chunks missing less than half of their redundancy filtered := make(map[uint64][]uint64) for chunk, pieces := range offline { if len(pieces) > f.erasureCode.NumPieces()/2 { filtered[chunk] = pieces } } return filtered } // threadedRepairLoop improves the health of files tracked by the renter by // reuploading their missing pieces. Multiple repair attempts may be necessary // before the file reaches full redundancy. func (r *Renter) threadedRepairLoop() { for { time.Sleep(5 * time.Second) if len(r.hostContractor.Contracts()) == 0 { // nothing to revise continue } // if the downloading flag is set, abort early. Otherwise, set the // uploading flag. id := r.mu.Lock() downloading := r.downloading if !downloading { r.uploading = true } r.mu.Unlock(id) if downloading { continue } // make copy of repair set under lock repairing := make(map[string]trackedFile) id = r.mu.RLock() for name, meta := range r.tracking { repairing[name] = meta } r.mu.RUnlock(id) // create host pool pool := r.newHostPool() for name, meta := range repairing { r.threadedRepairFile(name, meta, pool) } pool.Close() // heh // unset uploading flag id = r.mu.Lock() r.uploading = false r.mu.Unlock(id) } } // threadedRepairFile repairs and saves an individual file. func (r *Renter) threadedRepairFile(name string, meta trackedFile, pool *hostPool) { // helper function logAndRemove := func(fmt string, args ...interface{}) { r.log.Printf(fmt, args...) id := r.mu.Lock() delete(r.tracking, name) r.save() r.mu.Unlock(id) } id := r.mu.RLock() f, ok := r.files[name] r.mu.RUnlock(id) if !ok { logAndRemove("removing %v from repair set: no longer tracking that file", name) return } // determine if there is any work to do incChunks := f.incompleteChunks() if len(incChunks) == 0 { return } // open file handle handle, err := os.Open(meta.RepairPath) if err != nil { logAndRemove("removing %v from repair set: %v", name, err) return } defer handle.Close() // repair incomplete chunks if len(incChunks) != 0 { r.log.Printf("repairing %v chunks of %v", len(incChunks), f.name) r.repairChunks(f, handle, incChunks, pool) } } // repairChunks uploads missing chunks of f to new hosts. func (r *Renter) repairChunks(f *file, handle io.ReaderAt, chunks map[uint64][]uint64, pool *hostPool) { for chunk, pieces := range chunks { // Determine host set. We want one host for each missing piece, and no // repeats of other hosts of this chunk. hosts := pool.uniqueHosts(len(pieces), f.chunkHosts(chunk)) if len(hosts) == 0 { r.log.Debugf("aborting repair of %v: host pool is empty", f.name) return } // upload to new hosts err := f.repair(chunk, pieces, handle, hosts) if err != nil { if he, ok := err.(hostErrs); ok { // if a specific host failed, remove it from the pool for _, h := range he { // only log non-graceful errors if h.err != modules.ErrStopResponse { r.log.Printf("failed to upload to host %v: %v", h.host, h.err) } pool.remove(h.host) } } else { // any other type of error indicates a serious problem r.log.Printf("aborting repair of %v: %v", f.name, err) return } } // save the new contract f.mu.RLock() err = r.saveFile(f) f.mu.RUnlock() if err != nil { // If saving failed for this chunk, it will probably fail for the // next chunk as well. Better to try again on the next cycle. r.log.Printf("failed to save repaired file %v: %v", f.name, err) return } // check for download interruption id := r.mu.RLock() downloading := r.downloading r.mu.RUnlock(id) if downloading { return } } } Sia-1.0.4/modules/renter/repair_test.go000066400000000000000000000126141302033070500200440ustar00rootroot00000000000000package renter import ( "bytes" "crypto/rand" "errors" "reflect" "strconv" "sync" "testing" "time" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/renter/contractor" "github.com/NebulousLabs/Sia/types" ) // a testHost simulates a host. It implements the contractor.Editor interface. type testHost struct { ip modules.NetAddress sectors map[crypto.Hash][]byte // used to simulate real-world conditions delay time.Duration // transfers will take this long failRate int // transfers will randomly fail with probability 1/failRate sync.Mutex } // stub implementations of the contractor.Editor methods func (h *testHost) Address() modules.NetAddress { return h.ip } func (h *testHost) Delete(crypto.Hash) error { return nil } func (h *testHost) Modify(crypto.Hash, crypto.Hash, uint64, []byte) error { return nil } func (h *testHost) EndHeight() types.BlockHeight { return 0 } func (h *testHost) Close() error { return nil } // ContractID returns a fake (but unique) file contract ID. func (h *testHost) ContractID() types.FileContractID { var fcid types.FileContractID copy(fcid[:], h.ip) return fcid } // Upload adds a piece to the testHost. It randomly fails according to the // testHost's parameters. func (h *testHost) Upload(data []byte) (crypto.Hash, error) { // simulate I/O delay time.Sleep(h.delay) h.Lock() defer h.Unlock() // randomly fail if n, _ := crypto.RandIntn(h.failRate); n == 0 { return crypto.Hash{}, errors.New("no data") } root := crypto.MerkleRoot(data) h.sectors[root] = data return root, nil } // TestRepair tests the repair method of the file type. func TestRepair(t *testing.T) { if testing.Short() { t.SkipNow() } // generate data const dataSize = 777 data := make([]byte, dataSize) rand.Read(data) // create Reed-Solomon encoder rsc, err := NewRSCode(2, 10) if err != nil { t.Fatal(err) } // create hosts const pieceSize = 10 hosts := make([]contractor.Editor, rsc.NumPieces()) for i := range hosts { hosts[i] = &testHost{ sectors: make(map[crypto.Hash][]byte), ip: modules.NetAddress(strconv.Itoa(i)), delay: time.Duration(i) * time.Millisecond, failRate: 5, // 20% failure rate } } // make one host really slow hosts[0].(*testHost).delay = 100 * time.Millisecond // make one host always fail hosts[1].(*testHost).failRate = 1 // upload data to hosts f := newFile("foo", rsc, pieceSize, dataSize) r := bytes.NewReader(data) for chunk, pieces := range f.incompleteChunks() { err = f.repair(chunk, pieces, r, hosts) // hostErrs are non-fatal if _, ok := err.(hostErrs); ok { continue } else if err != nil { t.Fatal(err) } } // download data chunks := make([][][]byte, f.numChunks()) for i := uint64(0); i < f.numChunks(); i++ { chunks[i] = make([][]byte, rsc.NumPieces()) } for _, h := range hosts { contract, exists := f.contracts[h.ContractID()] if !exists { continue } for _, p := range contract.Pieces { encPiece := h.(*testHost).sectors[p.MerkleRoot] piece, err := deriveKey(f.masterKey, p.Chunk, p.Piece).DecryptBytes(encPiece) if err != nil { t.Fatal(err) } chunks[p.Chunk][p.Piece] = piece } } buf := new(bytes.Buffer) for _, chunk := range chunks { err = rsc.Recover(chunk, f.chunkSize(), buf) if err != nil { t.Fatal(err) } } buf.Truncate(dataSize) if !bytes.Equal(buf.Bytes(), data) { t.Fatal("recovered data does not match original") } /* // These metrics can be used to assess the efficiency of the repair // algorithm. for i, h := range hosts { host := h.(*testHost) pieces := 0 for _, p := range host.pieceMap { pieces += len(p) } t.Logf("Host #: %d\tDelay: %v\t# Pieces: %v\t# Chunks: %d", i, host.delay, pieces, len(host.pieceMap)) } */ } // offlineHostDB is a mocked hostDB, used for testing the offlineChunks method // of the file type. It is implemented as a map from NetAddresses to booleans, // where the bool indicates whether the host is active. type offlineHostDB struct { stubHostDB hosts map[modules.NetAddress]bool } // IsOffline is a stub implementation of the IsOffline method. func (hdb *offlineHostDB) IsOffline(addr modules.NetAddress) bool { return !hdb.hosts[addr] } // TestOfflineChunks tests the offlineChunks method of the file type. func TestOfflineChunks(t *testing.T) { // Create a mock hostdb. hdb := &offlineHostDB{ hosts: map[modules.NetAddress]bool{ "foo": false, "bar": false, "baz": true, }, } rsc, _ := NewRSCode(1, 1) f := &file{ erasureCode: rsc, contracts: map[types.FileContractID]fileContract{ {0}: {IP: "foo", Pieces: []pieceData{{0, 0, crypto.Hash{}}, {1, 0, crypto.Hash{}}}}, {1}: {IP: "bar", Pieces: []pieceData{{0, 1, crypto.Hash{}}}}, {2}: {IP: "baz", Pieces: []pieceData{{1, 1, crypto.Hash{}}}}, }, } // pieces 0.0, 0.1, and 1.0 are offline. Since redundancy is 1, // offlineChunks should report only chunk 0 as needing repair. expChunks := map[uint64][]uint64{ 0: {0, 1}, } chunks := f.offlineChunks(hdb) if !reflect.DeepEqual(chunks, expChunks) { // pieces may have been in a different order if !reflect.DeepEqual(chunks, map[uint64][]uint64{0: {1, 0}}) { t.Fatalf("offlineChunks did not return correct chunks: expected %v, got %v", expChunks, chunks) } } } Sia-1.0.4/modules/renter/upload.go000066400000000000000000000050351302033070500170060ustar00rootroot00000000000000package renter import ( "errors" "fmt" "os" "strings" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" ) var ( errInsufficientContracts = errors.New("not enough contracts to upload file") // Erasure-coded piece size pieceSize = modules.SectorSize - crypto.TwofishOverhead // defaultDataPieces is the number of data pieces per erasure-coded chunk defaultDataPieces = func() int { switch build.Release { case "dev": return 1 case "standard": return 4 case "testing": return 1 } panic("undefined defaultDataPieces") }() // defaultParityPieces is the number of parity pieces per erasure-coded // chunk defaultParityPieces = func() int { switch build.Release { case "dev": return 1 case "standard": return 20 case "testing": return 8 } panic("undefined defaultParityPieces") }() ) // Upload instructs the renter to start tracking a file. The renter will // automatically upload and repair tracked files using a background loop. func (r *Renter) Upload(up modules.FileUploadParams) error { // Enforce nickname rules. if strings.HasPrefix(up.SiaPath, "/") { return errors.New("nicknames cannot begin with /") } if up.SiaPath == "" { return ErrEmptyFilename } // Check for a nickname conflict. lockID := r.mu.RLock() _, exists := r.files[up.SiaPath] r.mu.RUnlock(lockID) if exists { return ErrPathOverload } // Fill in any missing upload params with sensible defaults. fileInfo, err := os.Stat(up.Source) if err != nil { return err } if up.ErasureCode == nil { up.ErasureCode, _ = NewRSCode(defaultDataPieces, defaultParityPieces) } // Check that we have contracts to upload to. We need at least (data + // parity/2) contracts; since NumPieces = data + parity, we arrive at the // expression below. if nContracts := len(r.hostContractor.Contracts()); nContracts < (up.ErasureCode.NumPieces()+up.ErasureCode.MinPieces())/2 && build.Release != "testing" { return fmt.Errorf("not enough contracts to upload file: got %v, needed %v", nContracts, (up.ErasureCode.NumPieces()+up.ErasureCode.MinPieces())/2) } // Create file object. f := newFile(up.SiaPath, up.ErasureCode, pieceSize, uint64(fileInfo.Size())) f.mode = uint32(fileInfo.Mode()) // Add file to renter. lockID = r.mu.Lock() r.files[up.SiaPath] = f r.tracking[up.SiaPath] = trackedFile{ RepairPath: up.Source, } r.saveSync() r.mu.Unlock(lockID) // Save the .sia file to the renter directory. err = r.saveFile(f) if err != nil { return err } return nil } Sia-1.0.4/modules/renter/upload_test.go000066400000000000000000000101211302033070500200350ustar00rootroot00000000000000package renter import ( "bytes" "io/ioutil" "path/filepath" "sync" "testing" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/renter/contractor" "github.com/NebulousLabs/Sia/types" ) // uploadDownloadContractor is a mocked hostContractor, contractor.Editor, and contractor.Downloader. It is // used for testing the uploading and downloading functions of the renter. type uploadDownloadContractor struct { stubContractor sectors map[crypto.Hash][]byte mu sync.Mutex } func (uc *uploadDownloadContractor) Contract(modules.NetAddress) (modules.RenterContract, bool) { return modules.RenterContract{}, true } func (uc *uploadDownloadContractor) Contracts() []modules.RenterContract { return make([]modules.RenterContract, 24) // exact number shouldn't matter, as long as its large enough } // Editor simply returns the uploadDownloadContractor, since it also implements the // Editor interface. func (uc *uploadDownloadContractor) Editor(types.FileContractID) (contractor.Editor, error) { return uc, nil } // Downloader simply returns the uploadDownloadContractor, since it also // implements the Downloader interface. func (uc *uploadDownloadContractor) Downloader(types.FileContractID) (contractor.Downloader, error) { return uc, nil } // Upload simulates a successful data upload. func (uc *uploadDownloadContractor) Upload(data []byte) (crypto.Hash, error) { uc.mu.Lock() defer uc.mu.Unlock() root := crypto.MerkleRoot(data) uc.sectors[root] = data return root, nil } // Download simulates a successful data download. func (uc *uploadDownloadContractor) Sector(root crypto.Hash) ([]byte, error) { uc.mu.Lock() defer uc.mu.Unlock() return uc.sectors[root], nil } // stub implementations of the contractor.Editor methods func (*uploadDownloadContractor) Address() modules.NetAddress { return "" } func (*uploadDownloadContractor) Delete(crypto.Hash) error { return nil } func (*uploadDownloadContractor) Modify(crypto.Hash, crypto.Hash, uint64, []byte) error { return nil } func (*uploadDownloadContractor) ContractID() types.FileContractID { return types.FileContractID{} } func (*uploadDownloadContractor) EndHeight() types.BlockHeight { return 10000 } func (*uploadDownloadContractor) Close() error { return nil } // TestUploadDownload tests the Upload and Download methods using a mock // contractor. func TestUploadDownload(t *testing.T) { if testing.Short() { t.SkipNow() } // create renter with mocked contractor hc := &uploadDownloadContractor{ sectors: make(map[crypto.Hash][]byte), } rt, err := newContractorTester("TestUploadDownload", nil, hc) if err != nil { t.Fatal(err) } // create a file data, err := crypto.RandBytes(777) if err != nil { t.Fatal(err) } source := filepath.Join(build.SiaTestingDir, "renter", "TestUploadDownload", "test.dat") err = ioutil.WriteFile(source, data, 0600) if err != nil { t.Fatal(err) } // use 1-1 erasure code, because we'll only have one host rsc, _ := NewRSCode(1, 1) // upload file err = rt.renter.Upload(modules.FileUploadParams{ Source: source, SiaPath: "foo", ErasureCode: rsc, // Upload will use sane defaults for other params }) if err != nil { t.Fatal(err) } files := rt.renter.FileList() if len(files) != 1 { t.Fatal("expected 1 file, got", len(files)) } // wait for repair loop for fully upload file for i := 0; i < 10 && !files[0].Available; i++ { files = rt.renter.FileList() time.Sleep(time.Second) } if !files[0].Available { t.Fatal("file did not reach full availability:", files[0].UploadProgress) } // download the file dest := filepath.Join(build.SiaTestingDir, "renter", "TestUploadDownload", "test.dat") err = rt.renter.Download("foo", dest) if err != nil { t.Fatal(err) } downData, err := ioutil.ReadFile(dest) if err != nil { t.Fatal(err) } if !bytes.Equal(downData, data) { t.Fatal("recovered data does not match original") } } Sia-1.0.4/modules/storagemanager.go000066400000000000000000000115001302033070500172140ustar00rootroot00000000000000package modules import ( "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/types" ) const ( // StorageManagerDir is standard name used for the directory that contains // all of the storage manager files. StorageManagerDir = "storagemanager" ) type ( // StorageFolderMetadata contains metadata about a storage folder that is // tracked by the storage folder manager. StorageFolderMetadata struct { Capacity uint64 `json:"capacity"` // bytes CapacityRemaining uint64 `json:"capacityremaining"` // bytes Path string `json:"path"` // Below are statistics about the filesystem. FailedReads and // FailedWrites are only incremented if the filesystem is returning // errors when operations are being performed. A large number of // FailedWrites can indicate that more space has been allocated on a // drive than is physically available. A high number of failures can // also indicate disk trouble. FailedReads uint64 `json:"failedreads"` FailedWrites uint64 `json:"failedwrites"` SuccessfulReads uint64 `json:"successfulreads"` SuccessfulWrites uint64 `json:"successfulwrites"` } // A StorageManager is responsible for managing storage folders and // sectors. Sectors are the base unit of storage that gets moved between // renters and hosts, and primarily is stored on the hosts. StorageManager interface { // AddSector will add a sector to the storage manager. If the sector // already exists, a virtual sector will be added, meaning that the // 'sectorData' will be ignored and no new disk space will be consumed. // The expiry height is used to track what height the sector can be // safely deleted at, though typically the host will manually delete // the sector before the expiry height. The same sector can be added // multiple times at different expiry heights, and the storage manager // is expected to only store the data once. AddSector(sectorRoot crypto.Hash, expiryHeight types.BlockHeight, sectorData []byte) error // AddSectorBatch is a performance optimization over AddSector when // adding a bunch of virtual sectors. It is necessary because otherwise // potentially thousands or even tens-of-thousands of fsync calls would // need to be made in serial, which would prevent renters from ever // successfully renewing. AddSectorBatch(sectorRoots []crypto.Hash, expiryHeight types.BlockHeight) error // AddStorageFolder adds a storage folder to the manager. The manager // may not check that there is enough space available on-disk to // support as much storage as requested, though the manager should // gracefully handle running out of storage unexpectedly. AddStorageFolder(path string, size uint64) error // The storage manager needs to be able to shut down. Close() error // DeleteSector deletes a sector, meaning that the manager will be // unable to upload that sector and be unable to provide a storage // proof on that sector. DeleteSector is for removing the data // entirely, and will remove instances of the sector appearing at all // heights. The primary purpose of DeleteSector is to comply with legal // requests to remove data. DeleteSector(sectorRoot crypto.Hash) error // ReadSector will read a sector from the storage manager, returning the // bytes that match the input sector root. ReadSector(sectorRoot crypto.Hash) ([]byte, error) // RemoveSector will remove a sector from the storage manager. The // height at which the sector expires should be provided, so that the // auto-expiry information for that sector can be properly updated. RemoveSector(sectorRoot crypto.Hash, expiryHeight types.BlockHeight) error // RemoveStorageFolder will remove a storage folder from the manager. // All storage on the folder will be moved to other storage folders, // meaning that no data will be lost. If the manager is unable to save // data, an error will be returned and the operation will be stopped. RemoveStorageFolder(index int, force bool) error // ResetStorageFolderHealth will reset the health statistics on a // storage folder. ResetStorageFolderHealth(index int) error // ResizeStorageFolder will grow or shrink a storage folder in the // manager. The manager may not check that there is enough space // on-disk to support growing the storage folder, but should gracefully // handle running out of space unexpectedly. When shrinking a storage // folder, any data in the folder that needs to be moved will be placed // into other storage folders, meaning that no data will be lost. If // the manager is unable to migrate the data, an error will be returned // and the operation will be stopped. ResizeStorageFolder(index int, newSize uint64) error // StorageFolders will return a list of storage folders tracked by the // manager. StorageFolders() []StorageFolderMetadata } ) Sia-1.0.4/modules/transactionpool.go000066400000000000000000000131471302033070500174450ustar00rootroot00000000000000package modules import ( "errors" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/types" ) const ( // TransactionSizeLimit defines the size of the largest transaction that // will be accepted by the transaction pool according to the IsStandard // rules. TransactionSizeLimit = 32e3 // TransactionSetSizeLimit defines the largest set of dependent unconfirmed // transactions that will be accepted by the transaction pool. TransactionSetSizeLimit = 250e3 ) var ( // ErrDuplicateTransactionSet is the error that gets returned if a // duplicate transaction set is given to the transaction pool. ErrDuplicateTransactionSet = errors.New("transaction set contains only duplicate transactions") // ErrLargeTransaction is the error that gets returned if a transaction // provided to the transaction pool is larger than what is allowed by the // IsStandard rules. ErrLargeTransaction = errors.New("transaction is too large for this transaction pool") // ErrLargeTransactionSet is the error that gets returned if a transaction // set given to the transaction pool is larger than the limit placed by the // IsStandard rules of the transaction pool. ErrLargeTransactionSet = errors.New("transaction set is too large for this transaction pool") // ErrInvalidArbPrefix is the error that gets returned if a transaction is // submitted to the transaction pool which contains a prefix that is not // recognized. This helps prevent miners on old versions from mining // potentially illegal transactions in the event of a soft-fork. ErrInvalidArbPrefix = errors.New("transaction contains non-standard arbitrary data") // PrefixNonSia defines the prefix that should be appended to any // transactions that use the arbitrary data for reasons outside of the // standard Sia protocol. This will prevent these transactions from being // rejected by the IsStandard set of rules, but also means that the data // will never be used within the formal Sia protocol. PrefixNonSia = types.Specifier{'N', 'o', 'n', 'S', 'i', 'a'} // TransactionPoolDir is the name of the directory that is used to store // the transaction pool's persistent data. TransactionPoolDir = "transactionpool" ) // A TransactionPoolSubscriber receives updates about the confirmed and // unconfirmed set from the transaction pool. Generally, there is no need to // subscribe to both the consensus set and the transaction pool. type TransactionPoolSubscriber interface { // ReceiveTransactionPoolUpdate notifies subscribers of a change to the // consensus set and/or unconfirmed set, and includes the consensus change // that would result if all of the transactions made it into a block. ReceiveUpdatedUnconfirmedTransactions([]types.Transaction, ConsensusChange) } // A TransactionPool manages unconfirmed transactions. type TransactionPool interface { // AcceptTransactionSet accepts a set of potentially interdependent // transactions. AcceptTransactionSet([]types.Transaction) error // Close is necessary for clean shutdown (e.g. during testing). Close() error // FeeEstimation returns an estimation for how high the transaction fee // needs to be per byte. The minimum recommended targets getting accepted // in ~3 blocks, and the maximum recommended targets getting accepted // immediately. Taking the average has a moderate chance of being accepted // within one block. The minimum has a strong chance of getting accepted // within 10 blocks. FeeEstimation() (minimumRecommended, maximumRecommended types.Currency) // IsStandardTransaction returns `err = nil` if the transaction is // standard, otherwise it returns an error explaining what is not standard. IsStandardTransaction(types.Transaction) error // PurgeTransactionPool is a temporary function available to the miner. In // the event that a miner mines an unacceptable block, the transaction pool // will be purged to clear out the transaction pool and get rid of the // illegal transaction. This should never happen, however there are bugs // that make this condition necessary. PurgeTransactionPool() // TransactionList returns a list of all transactions in the transaction // pool. The transactions are provided in an order that can acceptably be // put into a block. TransactionList() []types.Transaction // TransactionPoolSubscribe adds a subscriber to the transaction pool. // Subscribers will receive all consensus set changes as well as // transaction pool changes, and should not subscribe to both. TransactionPoolSubscribe(TransactionPoolSubscriber) // Unsubscribe removes a subscriber from the transaction pool. // This is necessary for clean shutdown of the miner. Unsubscribe(TransactionPoolSubscriber) } // ConsensusConflict implements the error interface, and indicates that a // transaction was rejected due to being incompatible with the current // consensus set, meaning either a double spend or a consensus rule violation - // it is unlikely that the transaction will ever be valid. type ConsensusConflict string // NewConsensusConflict returns a consensus conflict, which implements the // error interface. func NewConsensusConflict(s string) ConsensusConflict { return ConsensusConflict("consensus conflict: " + s) } // Error implements the error interface, turning the consensus conflict into an // acceptable error type. func (cc ConsensusConflict) Error() string { return string(cc) } // CalculateFee returns the fee-per-byte of a transaction set. func CalculateFee(ts []types.Transaction) types.Currency { var sum types.Currency for _, t := range ts { for _, fee := range t.MinerFees { sum = sum.Add(fee) } } size := len(encoding.Marshal(ts)) return sum.Div64(uint64(size)) } Sia-1.0.4/modules/transactionpool/000077500000000000000000000000001302033070500171105ustar00rootroot00000000000000Sia-1.0.4/modules/transactionpool/accept.go000066400000000000000000000253171302033070500207060ustar00rootroot00000000000000package transactionpool // TODO: It seems like the transaction pool is not properly detecting conflicts // between a file contract revision and a file contract. import ( "errors" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) const ( // The TransactionPoolSizeLimit is first checked, and then a transaction // set is added. The current transaction pool does not do any priority // ordering, so the size limit is such that the transaction pool will never // exceed the size of a block. // // TODO: Add a priority structure that will allow the transaction pool to // fill up beyond the size of a single block, without being subject to // manipulation. // // The first ~1/4 of the transaction pool can be filled for free. This is // mostly to preserve compatibility with clients that do not add fees. TransactionPoolSizeLimit = 2e6 - 5e3 - modules.TransactionSetSizeLimit TransactionPoolSizeForFee = 500e3 ) var ( errObjectConflict = errors.New("transaction set conflicts with an existing transaction set") errFullTransactionPool = errors.New("transaction pool cannot accept more transactions") errLowMinerFees = errors.New("transaction set needs more miner fees to be accepted") errEmptySet = errors.New("transaction set is empty") TransactionMinFee = types.SiacoinPrecision.Mul64(2) ) // relatedObjectIDs determines all of the object ids related to a transaction. func relatedObjectIDs(ts []types.Transaction) []ObjectID { oidMap := make(map[ObjectID]struct{}) for _, t := range ts { for _, sci := range t.SiacoinInputs { oidMap[ObjectID(sci.ParentID)] = struct{}{} } for i := range t.SiacoinOutputs { oidMap[ObjectID(t.SiacoinOutputID(uint64(i)))] = struct{}{} } for i := range t.FileContracts { oidMap[ObjectID(t.FileContractID(uint64(i)))] = struct{}{} } for _, fcr := range t.FileContractRevisions { oidMap[ObjectID(fcr.ParentID)] = struct{}{} } for _, sp := range t.StorageProofs { oidMap[ObjectID(sp.ParentID)] = struct{}{} } for _, sfi := range t.SiafundInputs { oidMap[ObjectID(sfi.ParentID)] = struct{}{} } for i := range t.SiafundOutputs { oidMap[ObjectID(t.SiafundOutputID(uint64(i)))] = struct{}{} } } var oids []ObjectID for oid := range oidMap { oids = append(oids, oid) } return oids } // checkMinerFees checks that the total amount of transaction fees in the // transaction set is sufficient to earn a spot in the transaction pool. func (tp *TransactionPool) checkMinerFees(ts []types.Transaction) error { // Transactions cannot be added after the TransactionPoolSizeLimit has been // hit. if tp.transactionListSize > TransactionPoolSizeLimit { return errFullTransactionPool } // The first TransactionPoolSizeForFee transactions do not need fees. if tp.transactionListSize > TransactionPoolSizeForFee { // Currently required fees are set on a per-transaction basis. 2 coins // are required per transaction if the free-fee limit has been reached, // adding a larger fee is not useful. var feeSum types.Currency for i := range ts { for _, fee := range ts[i].MinerFees { feeSum = feeSum.Add(fee) } } feeRequired := TransactionMinFee.Mul64(uint64(len(ts))) if feeSum.Cmp(feeRequired) < 0 { return errLowMinerFees } } return nil } // checkTransactionSetComposition checks if the transaction set is valid given // the state of the pool. It does not check that each individual transaction // would be legal in the next block, but does check things like miner fees and // IsStandard. func (tp *TransactionPool) checkTransactionSetComposition(ts []types.Transaction) error { // Check that the transaction set is not already known. setID := TransactionSetID(crypto.HashObject(ts)) _, exists := tp.transactionSets[setID] if exists { return modules.ErrDuplicateTransactionSet } // Check that the transaction set has enough fees to justify adding it to // the transaction list. err := tp.checkMinerFees(ts) if err != nil { return err } // All checks after this are expensive. // // TODO: There is no DoS prevention mechanism in place to prevent repeated // expensive verifications of invalid transactions that are created on the // fly. // Check that all transactions follow 'Standard.md' guidelines. err = tp.IsStandardTransactionSet(ts) if err != nil { return err } return nil } // handleConflicts detects whether the conflicts in the transaction pool are // legal children of the new transaction pool set or not. func (tp *TransactionPool) handleConflicts(ts []types.Transaction, conflicts []TransactionSetID) error { // Create a list of all the transaction ids that compose the set of // conflicts. conflictMap := make(map[types.TransactionID]TransactionSetID) for _, conflict := range conflicts { conflictSet := tp.transactionSets[conflict] for _, conflictTxn := range conflictSet { conflictMap[conflictTxn.ID()] = conflict } } // Discard all duplicate transactions from the input transaction set. var dedupSet []types.Transaction for _, t := range ts { _, exists := conflictMap[t.ID()] if exists { continue } dedupSet = append(dedupSet, t) } if len(dedupSet) == 0 { return modules.ErrDuplicateTransactionSet } // If transactions were pruned, it's possible that the set of // dependencies/conflicts has also reduced. To minimize computational load // on the consensus set, we want to prune out all of the conflicts that are // no longer relevant. As an example, consider the transaction set {A}, the // set {B}, and the new set {A, C}, where C is dependent on B. {A} and {B} // are both conflicts, but after deduplication {A} is no longer a conflict. // This is recursive, but it is guaranteed to run only once as the first // deduplication is guaranteed to be complete. if len(dedupSet) < len(ts) { oids := relatedObjectIDs(dedupSet) var conflicts []TransactionSetID for _, oid := range oids { conflict, exists := tp.knownObjects[oid] if exists { conflicts = append(conflicts, conflict) } } return tp.handleConflicts(dedupSet, conflicts) } // Merge all of the conflict sets with the input set (input set goes last // to preserve dependency ordering), and see if the set as a whole is both // small enough to be legal and valid as a set. If no, return an error. If // yes, add the new set to the pool, and eliminate the old set. The output // diff objects can be repeated, (no need to remove those). Just need to // remove the conflicts from tp.transactionSets. var superset []types.Transaction supersetMap := make(map[TransactionSetID]struct{}) for _, conflict := range conflictMap { supersetMap[conflict] = struct{}{} } for conflict := range supersetMap { superset = append(superset, tp.transactionSets[conflict]...) } superset = append(superset, dedupSet...) // Check the composition of the transaction set, including fees and // IsStandard rules (this is a new set, the rules must be rechecked). err := tp.checkTransactionSetComposition(superset) if err != nil { return err } // Check that the transaction set is valid. cc, err := tp.consensusSet.TryTransactionSet(superset) if err != nil { return modules.NewConsensusConflict(err.Error()) } // Remove the conflicts from the transaction pool. The diffs do not need to // be removed, they will be overwritten later in the function. for _, conflict := range conflictMap { conflictSet := tp.transactionSets[conflict] tp.transactionListSize -= len(encoding.Marshal(conflictSet)) delete(tp.transactionSets, conflict) delete(tp.transactionSetDiffs, conflict) } // Add the transaction set to the pool. setID := TransactionSetID(crypto.HashObject(superset)) tp.transactionSets[setID] = superset for _, diff := range cc.SiacoinOutputDiffs { tp.knownObjects[ObjectID(diff.ID)] = setID } for _, diff := range cc.FileContractDiffs { tp.knownObjects[ObjectID(diff.ID)] = setID } for _, diff := range cc.SiafundOutputDiffs { tp.knownObjects[ObjectID(diff.ID)] = setID } tp.transactionSetDiffs[setID] = cc tp.transactionListSize += len(encoding.Marshal(superset)) return nil } // acceptTransactionSet verifies that a transaction set is allowed to be in the // transaction pool, and then adds it to the transaction pool. func (tp *TransactionPool) acceptTransactionSet(ts []types.Transaction) error { if len(ts) == 0 { return errEmptySet } // Remove all transactions that have been confirmed in the transaction set. err := tp.db.Update(func(tx *bolt.Tx) error { oldTS := ts ts = []types.Transaction{} for _, txn := range oldTS { if !tp.transactionConfirmed(tx, txn.ID()) { ts = append(ts, txn) } } return nil }) if err != nil { return err } // If no transactions remain, return a dublicate error. if len(ts) == 0 { return modules.ErrDuplicateTransactionSet } // Check the composition of the transaction set, including fees and // IsStandard rules. err = tp.checkTransactionSetComposition(ts) if err != nil { return err } // Check for conflicts with other transactions, which would indicate a // double-spend. Legal children of a transaction set will also trigger the // conflict-detector. oids := relatedObjectIDs(ts) var conflicts []TransactionSetID for _, oid := range oids { conflict, exists := tp.knownObjects[oid] if exists { conflicts = append(conflicts, conflict) } } if len(conflicts) > 0 { return tp.handleConflicts(ts, conflicts) } cc, err := tp.consensusSet.TryTransactionSet(ts) if err != nil { return modules.NewConsensusConflict(err.Error()) } // Add the transaction set to the pool. setID := TransactionSetID(crypto.HashObject(ts)) tp.transactionSets[setID] = ts for _, oid := range oids { tp.knownObjects[oid] = setID } tp.transactionSetDiffs[setID] = cc tp.transactionListSize += len(encoding.Marshal(ts)) return nil } // AcceptTransaction adds a transaction to the unconfirmed set of // transactions. If the transaction is accepted, it will be relayed to // connected peers. func (tp *TransactionPool) AcceptTransactionSet(ts []types.Transaction) error { tp.mu.Lock() defer tp.mu.Unlock() err := tp.acceptTransactionSet(ts) if err != nil { return err } // Notify subscribers and broadcast the transaction set. go tp.gateway.Broadcast("RelayTransactionSet", ts, tp.gateway.Peers()) tp.updateSubscribersTransactions() return nil } // relayTransactionSet is an RPC that accepts a transaction set from a peer. If // the accept is successful, the transaction will be relayed to the gateway's // other peers. func (tp *TransactionPool) relayTransactionSet(conn modules.PeerConn) error { var ts []types.Transaction err := encoding.ReadObject(conn, &ts, types.BlockSizeLimit) if err != nil { return err } return tp.AcceptTransactionSet(ts) } Sia-1.0.4/modules/transactionpool/accept_test.go000066400000000000000000000425511302033070500217440ustar00rootroot00000000000000package transactionpool import ( "crypto/rand" "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // TestIntegrationAcceptTransactionSet probes the AcceptTransactionSet method // of the transaction pool. func TestIntegrationAcceptTransactionSet(t *testing.T) { // Create a transaction pool tester. tpt, err := createTpoolTester("TestIntegrationAcceptTransactionSet") if err != nil { t.Fatal(err) } defer tpt.Close() // Check that the transaction pool is empty. if len(tpt.tpool.transactionSets) != 0 { t.Error("transaction pool is not empty") } // Create a valid transaction set using the wallet. txns, err := tpt.wallet.SendSiacoins(types.NewCurrency64(100), types.UnlockHash{}) if err != nil { t.Fatal(err) } if len(tpt.tpool.transactionSets) != 1 { t.Error("sending coins did not increase the transaction sets by 1") } // Submit the transaction set again to trigger a duplication error. err = tpt.tpool.AcceptTransactionSet(txns) if err != modules.ErrDuplicateTransactionSet { t.Error(err) } // Mine a block and check that the transaction pool gets emptied. block, _ := tpt.miner.FindBlock() err = tpt.cs.AcceptBlock(block) if err != nil { t.Fatal(err) } if len(tpt.tpool.TransactionList()) != 0 { t.Error("transaction pool was not emptied after mining a block") } // Try to resubmit the transaction set to verify err = tpt.tpool.AcceptTransactionSet(txns) if err == nil { t.Error("transaction set was supposed to be rejected") } } // TestIntegrationConflictingTransactionSets tries to add two transaction sets // to the transaction pool that are each legal individually, but double spend // an output. func TestIntegrationConflictingTransactionSets(t *testing.T) { if testing.Short() { t.SkipNow() } // Create a transaction pool tester. tpt, err := createTpoolTester("TestIntegrationConflictingTransactionSets") if err != nil { t.Fatal(err) } defer tpt.Close() // Fund a partial transaction. fund := types.NewCurrency64(30e6) txnBuilder := tpt.wallet.StartTransaction() err = txnBuilder.FundSiacoins(fund) if err != nil { t.Fatal(err) } // wholeTransaction is set to false so that we can use the same signature // to create a double spend. txnSet, err := txnBuilder.Sign(false) if err != nil { t.Fatal(err) } txnSetDoubleSpend := make([]types.Transaction, len(txnSet)) copy(txnSetDoubleSpend, txnSet) // There are now two sets of transactions that are signed and ready to // spend the same output. Have one spend the money in a miner fee, and the // other create a siacoin output. txnIndex := len(txnSet) - 1 txnSet[txnIndex].MinerFees = append(txnSet[txnIndex].MinerFees, fund) txnSetDoubleSpend[txnIndex].SiacoinOutputs = append(txnSetDoubleSpend[txnIndex].SiacoinOutputs, types.SiacoinOutput{Value: fund}) // Add the first and then the second txn set. err = tpt.tpool.AcceptTransactionSet(txnSet) if err != nil { t.Error(err) } err = tpt.tpool.AcceptTransactionSet(txnSetDoubleSpend) if err == nil { t.Error("transaction should not have passed inspection") } // Purge and try the sets in the reverse order. tpt.tpool.PurgeTransactionPool() err = tpt.tpool.AcceptTransactionSet(txnSetDoubleSpend) if err != nil { t.Error(err) } err = tpt.tpool.AcceptTransactionSet(txnSet) if err == nil { t.Error("transaction should not have passed inspection") } } // TestIntegrationCheckMinerFees probes the checkMinerFees method of the // transaction pool. func TestIntegrationCheckMinerFees(t *testing.T) { if testing.Short() { t.SkipNow() } // Create a transaction pool tester. tpt, err := createTpoolTester("TestIntegrationCheckMinerFees") if err != nil { t.Fatal(err) } defer tpt.Close() // Fill the transaction pool to the fee limit. for i := 0; i < TransactionPoolSizeForFee/10e3; i++ { arbData := make([]byte, 10e3) copy(arbData, modules.PrefixNonSia[:]) _, err = rand.Read(arbData[100:116]) // prevents collisions with other transacitons in the loop. if err != nil { t.Fatal(err) } txn := types.Transaction{ArbitraryData: [][]byte{arbData}} err := tpt.tpool.AcceptTransactionSet([]types.Transaction{txn}) if err != nil { t.Fatal(err) } } // Add another transaction, this one should fail for having too few fees. err = tpt.tpool.AcceptTransactionSet([]types.Transaction{{}}) if err != errLowMinerFees { t.Error(err) } // Add a transaction that has sufficient fees. _, err = tpt.wallet.SendSiacoins(types.NewCurrency64(100), types.UnlockHash{}) if err != nil { t.Error(err) } // TODO: fill the pool up all the way and try again. } // TestTransactionSuperset submits a single transaction to the network, // followed by a transaction set containing that single transaction. func TestIntegrationTransactionSuperset(t *testing.T) { if testing.Short() { t.SkipNow() } // Create a transaction pool tester. tpt, err := createTpoolTester("TestTransactionSuperset") if err != nil { t.Fatal(err) } defer tpt.Close() // Fund a partial transaction. fund := types.NewCurrency64(30e6) txnBuilder := tpt.wallet.StartTransaction() err = txnBuilder.FundSiacoins(fund) if err != nil { t.Fatal(err) } txnBuilder.AddMinerFee(fund) // wholeTransaction is set to false so that we can use the same signature // to create a double spend. txnSet, err := txnBuilder.Sign(false) if err != nil { t.Fatal(err) } if len(txnSet) <= 1 { t.Fatal("test is invalid unless the transaction set has two or more transactions") } // Check that the second transaction is dependent on the first. err = tpt.tpool.AcceptTransactionSet(txnSet[1:]) if err == nil { t.Fatal("transaction set must have dependent transactions") } // Submit the first transaction in the set to the transaction pool, and // then the superset. err = tpt.tpool.AcceptTransactionSet(txnSet[:1]) if err != nil { t.Fatal("first transaction in the transaction set was not valid?") } err = tpt.tpool.AcceptTransactionSet(txnSet) if err != nil { t.Fatal("super setting is not working:", err) } // Try resubmitting the individual transaction and the superset, a // duplication error should be returned for each case. err = tpt.tpool.AcceptTransactionSet(txnSet[:1]) if err != modules.ErrDuplicateTransactionSet { t.Fatal(err) } err = tpt.tpool.AcceptTransactionSet(txnSet) if err != modules.ErrDuplicateTransactionSet { t.Fatal("super setting is not working:", err) } } // TestTransactionSubset submits a transaction set to the network, followed by // just a subset, expectint ErrDuplicateTransactionSet as a response. func TestIntegrationTransactionSubset(t *testing.T) { if testing.Short() { t.SkipNow() } // Create a transaction pool tester. tpt, err := createTpoolTester("TestTransactionSubset") if err != nil { t.Fatal(err) } defer tpt.Close() // Fund a partial transaction. fund := types.NewCurrency64(30e6) txnBuilder := tpt.wallet.StartTransaction() err = txnBuilder.FundSiacoins(fund) if err != nil { t.Fatal(err) } txnBuilder.AddMinerFee(fund) // wholeTransaction is set to false so that we can use the same signature // to create a double spend. txnSet, err := txnBuilder.Sign(false) if err != nil { t.Fatal(err) } if len(txnSet) <= 1 { t.Fatal("test is invalid unless the transaction set has two or more transactions") } // Check that the second transaction is dependent on the first. err = tpt.tpool.AcceptTransactionSet(txnSet[1:]) if err == nil { t.Fatal("transaction set must have dependent transactions") } // Submit the set to the pool, followed by just the transaction. err = tpt.tpool.AcceptTransactionSet(txnSet) if err != nil { t.Fatal("super setting is not working:", err) } err = tpt.tpool.AcceptTransactionSet(txnSet[:1]) if err != modules.ErrDuplicateTransactionSet { t.Fatal(err) } } // TestIntegrationTransactionChild submits a single transaction to the network, // followed by a child transaction. func TestIntegrationTransactionChild(t *testing.T) { if testing.Short() { t.SkipNow() } // Create a transaction pool tester. tpt, err := createTpoolTester("TestTransactionChild") if err != nil { t.Fatal(err) } defer tpt.Close() // Fund a partial transaction. fund := types.NewCurrency64(30e6) txnBuilder := tpt.wallet.StartTransaction() err = txnBuilder.FundSiacoins(fund) if err != nil { t.Fatal(err) } txnBuilder.AddMinerFee(fund) // wholeTransaction is set to false so that we can use the same signature // to create a double spend. txnSet, err := txnBuilder.Sign(false) if err != nil { t.Fatal(err) } if len(txnSet) <= 1 { t.Fatal("test is invalid unless the transaction set has two or more transactions") } // Check that the second transaction is dependent on the first. err = tpt.tpool.AcceptTransactionSet([]types.Transaction{txnSet[1]}) if err == nil { t.Fatal("transaction set must have dependent transactions") } // Submit the first transaction in the set to the transaction pool. err = tpt.tpool.AcceptTransactionSet(txnSet[:1]) if err != nil { t.Fatal("first transaction in the transaction set was not valid?") } err = tpt.tpool.AcceptTransactionSet(txnSet[1:]) if err != nil { t.Fatal("child transaction not seen as valid") } } // TestIntegrationNilAccept tries submitting a nil transaction set and a 0-len // transaction set to the transaction pool. func TestIntegrationNilAccept(t *testing.T) { if testing.Short() { t.SkipNow() } tpt, err := createTpoolTester("TestTransactionChild") if err != nil { t.Fatal(err) } defer tpt.Close() err = tpt.tpool.AcceptTransactionSet(nil) if err == nil { t.Error("no error returned when submitting nothing to the transaction pool") } err = tpt.tpool.AcceptTransactionSet([]types.Transaction{}) if err == nil { t.Error("no error returned when submitting nothing to the transaction pool") } } // TestAcceptFCAndConflictingRevision checks that the transaction pool // correctly accepts a file contract in a transaction set followed by a correct // revision to that file contract in the a following transaction set, with no // block separating them. func TestAcceptFCAndConflictingRevision(t *testing.T) { if testing.Short() { t.SkipNow() } tpt, err := createTpoolTester("TestAcceptFCAndConflictingRevision") if err != nil { t.Fatal(err) } defer tpt.Close() // Create and fund a valid file contract. builder := tpt.wallet.StartTransaction() payout := types.NewCurrency64(1e9) err = builder.FundSiacoins(payout) if err != nil { t.Fatal(err) } builder.AddFileContract(types.FileContract{ WindowStart: tpt.cs.Height() + 2, WindowEnd: tpt.cs.Height() + 5, Payout: payout, ValidProofOutputs: []types.SiacoinOutput{{Value: types.PostTax(tpt.cs.Height(), payout)}}, MissedProofOutputs: []types.SiacoinOutput{{Value: types.PostTax(tpt.cs.Height(), payout)}}, UnlockHash: types.UnlockConditions{}.UnlockHash(), }) tSet, err := builder.Sign(true) if err != nil { t.Fatal(err) } err = tpt.tpool.AcceptTransactionSet(tSet) if err != nil { t.Fatal(err) } fcid := tSet[len(tSet)-1].FileContractID(0) // Create a file contract revision and submit it. rSet := []types.Transaction{{ FileContractRevisions: []types.FileContractRevision{{ ParentID: fcid, NewRevisionNumber: 2, NewWindowStart: tpt.cs.Height() + 2, NewWindowEnd: tpt.cs.Height() + 5, NewValidProofOutputs: []types.SiacoinOutput{{Value: types.PostTax(tpt.cs.Height(), payout)}}, NewMissedProofOutputs: []types.SiacoinOutput{{Value: types.PostTax(tpt.cs.Height(), payout)}}, }}, }} err = tpt.tpool.AcceptTransactionSet(rSet) if err != nil { t.Fatal(err) } } // TestPartialConfirmation checks that the transaction pool correctly accepts a // transaction set which has parents that have been accepted by the consensus // set but not the whole set has been accepted by the consensus set. func TestPartialConfirmation(t *testing.T) { if testing.Short() { t.SkipNow() } tpt, err := createTpoolTester("TestPartialConfirmation") if err != nil { t.Fatal(err) } defer tpt.Close() // Create and fund a valid file contract. builder := tpt.wallet.StartTransaction() payout := types.NewCurrency64(1e9) err = builder.FundSiacoins(payout) if err != nil { t.Fatal(err) } builder.AddFileContract(types.FileContract{ WindowStart: tpt.cs.Height() + 2, WindowEnd: tpt.cs.Height() + 5, Payout: payout, ValidProofOutputs: []types.SiacoinOutput{{Value: types.PostTax(tpt.cs.Height(), payout)}}, MissedProofOutputs: []types.SiacoinOutput{{Value: types.PostTax(tpt.cs.Height(), payout)}}, UnlockHash: types.UnlockConditions{}.UnlockHash(), }) tSet, err := builder.Sign(true) if err != nil { t.Fatal(err) } fcid := tSet[len(tSet)-1].FileContractID(0) // Create a file contract revision. rSet := []types.Transaction{{ FileContractRevisions: []types.FileContractRevision{{ ParentID: fcid, NewRevisionNumber: 2, NewWindowStart: tpt.cs.Height() + 2, NewWindowEnd: tpt.cs.Height() + 5, NewValidProofOutputs: []types.SiacoinOutput{{Value: types.PostTax(tpt.cs.Height(), payout)}}, NewMissedProofOutputs: []types.SiacoinOutput{{Value: types.PostTax(tpt.cs.Height(), payout)}}, }}, }} // Combine the contract and revision in to a single set. fullSet := append(tSet, rSet...) // Get the tSet onto the blockchain. unsolvedBlock, target, err := tpt.miner.BlockForWork() if err != nil { t.Fatal(err) } unsolvedBlock.Transactions = append(unsolvedBlock.Transactions, tSet...) solvedBlock, solved := tpt.miner.SolveBlock(unsolvedBlock, target) if !solved { t.Fatal("Failed to solve block") } err = tpt.cs.AcceptBlock(solvedBlock) if err != nil { t.Fatal(err) } // Try to get the full set into the transaction pool. The transaction pool // should recognize that the set is partially accepted, and be able to // accept on the the transactions that are new and are not yet on the // blockchain. err = tpt.tpool.AcceptTransactionSet(fullSet) if err != nil { t.Fatal(err) } } // TestPartialConfirmationWeave checks that the transaction pool correctly // accepts a transaction set which has parents that have been accepted by the // consensus set but not the whole set has been accepted by the consensus set, // this time weaving the dependencies, such that the first transaction is not // in the consensus set, the second is, and the third has both as dependencies. func TestPartialConfirmationWeave(t *testing.T) { if testing.Short() { t.SkipNow() } tpt, err := createTpoolTester("TestPartialConfirmation") if err != nil { t.Fatal(err) } defer tpt.Close() // Step 1: create an output to the empty address in a tx. // Step 2: create a second output to the empty address in another tx. // Step 3: create a transaction using both those outputs. // Step 4: mine the txn set in step 2 // Step 5: Submit the complete set. // Create a transaction with a single output to a fully controlled address. emptyUH := types.UnlockConditions{}.UnlockHash() builder1 := tpt.wallet.StartTransaction() funding1 := types.NewCurrency64(1e9) err = builder1.FundSiacoins(funding1) if err != nil { t.Fatal(err) } scOutput1 := types.SiacoinOutput{ Value: funding1, UnlockHash: emptyUH, } i1 := builder1.AddSiacoinOutput(scOutput1) tSet1, err := builder1.Sign(true) if err != nil { t.Fatal(err) } // Submit to the transaction pool and mine the block, to minimize // complexity. err = tpt.tpool.AcceptTransactionSet(tSet1) if err != nil { t.Fatal(err) } _, err = tpt.miner.AddBlock() if err != nil { t.Fatal(err) } // Create a second output to the fully controlled address, to fund the // second transaction in the weave. builder2 := tpt.wallet.StartTransaction() funding2 := types.NewCurrency64(2e9) err = builder2.FundSiacoins(funding2) if err != nil { t.Fatal(err) } scOutput2 := types.SiacoinOutput{ Value: funding2, UnlockHash: emptyUH, } i2 := builder2.AddSiacoinOutput(scOutput2) tSet2, err := builder2.Sign(true) if err != nil { t.Fatal(err) } // Submit to the transaction pool and mine the block, to minimize // complexity. err = tpt.tpool.AcceptTransactionSet(tSet2) if err != nil { t.Fatal(err) } _, err = tpt.miner.AddBlock() if err != nil { t.Fatal(err) } // Create a passthrough transaction for output1 and output2, so that they // can be used as unconfirmed dependencies. txn1 := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: tSet1[len(tSet1)-1].SiacoinOutputID(i1), }}, SiacoinOutputs: []types.SiacoinOutput{{ Value: funding1, UnlockHash: emptyUH, }}, } txn2 := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: tSet2[len(tSet2)-1].SiacoinOutputID(i2), }}, SiacoinOutputs: []types.SiacoinOutput{{ Value: funding2, UnlockHash: emptyUH, }}, } // Create a child transaction that depends on inputs from both txn1 and // txn2. child := types.Transaction{ SiacoinInputs: []types.SiacoinInput{ { ParentID: txn1.SiacoinOutputID(0), }, { ParentID: txn2.SiacoinOutputID(0), }, }, SiacoinOutputs: []types.SiacoinOutput{{ Value: funding1.Add(funding2), }}, } // Get txn2 accepted into the consensus set. err = tpt.tpool.AcceptTransactionSet([]types.Transaction{txn2}) if err != nil { t.Fatal(err) } _, err = tpt.miner.AddBlock() if err != nil { t.Fatal(err) } // Try to get the set of txn1, txn2, and child accepted into the // transaction pool. err = tpt.tpool.AcceptTransactionSet([]types.Transaction{txn1, txn2, child}) if err != nil { t.Fatal(err) } } Sia-1.0.4/modules/transactionpool/persist.go000066400000000000000000000105311302033070500211300ustar00rootroot00000000000000package transactionpool import ( "errors" "os" "path/filepath" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) var ( // bucketRecentConsensusChange holds the most recent consensus change seen // by the transaction pool. bucketRecentConsensusChange = []byte("RecentConsensusChange") // bucketConfirmedTransactions holds the ids of every transaction that has // been confirmed on the blockchain. bucketConfirmedTransactions = []byte("ConfirmedTransactions") // errNilConsensusChange is returned if there is no consensus change in the // database. errNilConsensusChange = errors.New("no consensus change found") // fieldRecentConsensusChange is the field in bucketRecentConsensusChange // that holds the value of the most recent consensus change. fieldRecentConsensusChange = []byte("RecentConsensusChange") ) // resetDB deletes all consensus related persistence from the transaction pool. func (tp *TransactionPool) resetDB(tx *bolt.Tx) error { err := tx.DeleteBucket(bucketConfirmedTransactions) if err != nil { return err } err = tp.putRecentConsensusChange(tx, modules.ConsensusChangeBeginning) if err != nil { return err } _, err = tx.CreateBucket(bucketConfirmedTransactions) return err } // initPersist creates buckets in the database func (tp *TransactionPool) initPersist() error { // Create the persist directory if it does not yet exist. err := os.MkdirAll(tp.persistDir, 0700) if err != nil { return err } // Open the database file. tp.db, err = persist.OpenDatabase(dbMetadata, filepath.Join(tp.persistDir, dbFilename)) if err != nil { return err } // Create the database and get the most recent consensus change. var cc modules.ConsensusChangeID err = tp.db.Update(func(tx *bolt.Tx) error { // Create the database buckets. buckets := [][]byte{ bucketRecentConsensusChange, bucketConfirmedTransactions, } for _, bucket := range buckets { _, err := tx.CreateBucketIfNotExists(bucket) if err != nil { return err } } // Get the recent consensus change. cc, err = tp.getRecentConsensusChange(tx) if err == errNilConsensusChange { return tp.putRecentConsensusChange(tx, modules.ConsensusChangeBeginning) } return err }) if err != nil { return err } // Subscribe to the consensus set using the most recent consensus change. err = tp.consensusSet.ConsensusSetSubscribe(tp, cc) if err == modules.ErrInvalidConsensusChangeID { // Reset and rescan because the consensus set does not recognize the // provided consensus change id. resetErr := tp.db.Update(func(tx *bolt.Tx) error { return tp.resetDB(tx) }) if resetErr != nil { return resetErr } return tp.consensusSet.ConsensusSetSubscribe(tp, modules.ConsensusChangeBeginning) } return err } // getRecentConsensusChange returns the most recent consensus change from the // database. func (tp *TransactionPool) getRecentConsensusChange(tx *bolt.Tx) (cc modules.ConsensusChangeID, err error) { ccBytes := tx.Bucket(bucketRecentConsensusChange).Get(fieldRecentConsensusChange) if ccBytes == nil { return modules.ConsensusChangeID{}, errNilConsensusChange } copy(cc[:], ccBytes) return cc, nil } // putRecentConsensusChange updates the most recent consensus change seen by // the transaction pool. func (tp *TransactionPool) putRecentConsensusChange(tx *bolt.Tx, cc modules.ConsensusChangeID) error { return tx.Bucket(bucketRecentConsensusChange).Put(fieldRecentConsensusChange, cc[:]) } // transactionConfirmed returns true if the transaction has been confirmed on // the blockchain and false if the transaction has not been confirmed on the // blockchain. func (tp *TransactionPool) transactionConfirmed(tx *bolt.Tx, id types.TransactionID) bool { confirmedBytes := tx.Bucket(bucketConfirmedTransactions).Get(id[:]) if confirmedBytes == nil { return false } return true } // addTransaction adds a transaction to the list of confirmed transactions. func (tp *TransactionPool) addTransaction(tx *bolt.Tx, id types.TransactionID) error { return tx.Bucket(bucketConfirmedTransactions).Put(id[:], []byte{}) } // deleteTransaction deletes a transaction from the list of confirmed // transactions. func (tp *TransactionPool) deleteTransaction(tx *bolt.Tx, id types.TransactionID) error { return tx.Bucket(bucketConfirmedTransactions).Delete(id[:]) } Sia-1.0.4/modules/transactionpool/persist_test.go000066400000000000000000000047331302033070500221760ustar00rootroot00000000000000package transactionpool import ( "os" "path/filepath" "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // TestRescan triggers a rescan in the transaction pool, verifying that the // rescan code does not cause deadlocks or crashes. func TestRescan(t *testing.T) { if testing.Short() { t.SkipNow() } tpt, err := createTpoolTester("TestRescan") if err != nil { t.Fatal(err) } defer tpt.Close() // Create a valid transaction set using the wallet. txns, err := tpt.wallet.SendSiacoins(types.NewCurrency64(100), types.UnlockHash{}) if err != nil { t.Fatal(err) } if len(tpt.tpool.transactionSets) != 1 { t.Error("sending coins did not increase the transaction sets by 1") } // Mine the transaction into a block, so that it's in the consensus set. _, err = tpt.miner.AddBlock() if err != nil { t.Fatal(err) } // Close the tpool, delete the persistence, then restart the tpool. The // tpool should still recognize the transaction set as a duplicate. persistDir := tpt.tpool.persistDir err = tpt.tpool.Close() if err != nil { t.Fatal(err) } err = os.RemoveAll(persistDir) if err != nil { t.Fatal(err) } tpt.tpool, err = New(tpt.cs, tpt.gateway, persistDir) if err != nil { t.Fatal(err) } err = tpt.tpool.AcceptTransactionSet(txns) if err != modules.ErrDuplicateTransactionSet { t.Fatal("expecting modules.ErrDuplicateTransactionSet, got:", err) } // Close the tpool, corrupt the database, then restart the tpool. The tpool // should still recognize the transaction set as a duplicate. err = tpt.tpool.Close() if err != nil { t.Fatal(err) } db, err := persist.OpenDatabase(dbMetadata, filepath.Join(persistDir, dbFilename)) if err != nil { t.Fatal(err) } err = db.Update(func(tx *bolt.Tx) error { ccBytes := tx.Bucket(bucketRecentConsensusChange).Get(fieldRecentConsensusChange) // copy the bytes due to bolt's mmap. newCCBytes := make([]byte, len(ccBytes)) copy(newCCBytes, ccBytes) newCCBytes[0]++ return tx.Bucket(bucketRecentConsensusChange).Put(fieldRecentConsensusChange, newCCBytes) }) if err != nil { t.Fatal(err) } err = db.Close() if err != nil { t.Fatal(err) } tpt.tpool, err = New(tpt.cs, tpt.gateway, persistDir) if err != nil { t.Fatal(err) } err = tpt.tpool.AcceptTransactionSet(txns) if err != modules.ErrDuplicateTransactionSet { t.Fatal("expecting modules.ErrDuplicateTransactionSet, got:", err) } } Sia-1.0.4/modules/transactionpool/standard.go000066400000000000000000000121101302033070500212320ustar00rootroot00000000000000package transactionpool import ( "errors" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // standard.go adds extra rules to transactions which help preserve network // health and provides flexibility for future soft forks and tweaks to the // network. // // Rule: Transaction size is limited // There is a DoS vector where large transactions can both contain many // signatures, and have each signature's CoveredFields object cover a // unique but large portion of the transaciton. A 1mb transaction could // force a verifier to hash very large volumes of data, which takes a long // time on nonspecialized hardware. // // Rule: Foreign signature algorithms are rejected. // There are plans to add newer, faster signature algorithms to Sia as the // project matures and the need for increased verification speed grows. // Foreign signatures are allowed into the blockchain, where they are // accepted as valid. Hoewver, if there has been a soft-fork, the foreign // signatures might actually be invalid. This rule protects legacy miners // from including potentially invalid transactions in their blocks. // // Rule: The types of allowed arbitrary data are limited // The arbitrary data field can be used to orchestrate soft-forks to Sia // that add features. Legacy miners are at risk of creating invalid blocks // if they include arbitrary data which has meanings that the legacy miner // doesn't understand. // // Rule: The transaction set size is limited. // A group of dependent transactions cannot exceed 100kb to limit how // quickly the transaction pool can be filled with new transactions. // checkUnlockConditions looks at the UnlockConditions and verifies that all // public keys are recognized. Unrecognized public keys are automatically // accepted as valid by the consnensus set, but rejected by the transaction // pool. This allows new types of keys to be added via a softfork without // alienating all of the older nodes. func (tp *TransactionPool) checkUnlockConditions(uc types.UnlockConditions) error { for _, pk := range uc.PublicKeys { if pk.Algorithm != types.SignatureEntropy && pk.Algorithm != types.SignatureEd25519 { return errors.New("unrecognized key type in transaction") } } return nil } // IsStandardTransaction enforces extra rules such as a transaction size limit. // These rules can be altered without disrupting consensus. func (tp *TransactionPool) IsStandardTransaction(t types.Transaction) error { // Check that the size of the transaction does not exceed the standard // established in Standard.md. Larger transactions are a DOS vector, // because someone can fill a large transaction with a bunch of signatures // that require hashing the entire transaction. Several hundred megabytes // of hashing can be required of a verifier. Enforcing this rule makes it // more difficult for attackers to exploid this DOS vector, though a miner // with sufficient power could still create unfriendly blocks. if len(encoding.Marshal(t)) > modules.TransactionSizeLimit { return modules.ErrLargeTransaction } // Check that all public keys are of a recognized type. Need to check all // of the UnlockConditions, which currently can appear in 3 separate fields // of the transaction. Unrecognized types are ignored because a softfork // may make certain unrecognized signatures invalid, and this node cannot // tell which sigantures are the invalid ones. for _, sci := range t.SiacoinInputs { err := tp.checkUnlockConditions(sci.UnlockConditions) if err != nil { return err } } for _, fcr := range t.FileContractRevisions { err := tp.checkUnlockConditions(fcr.UnlockConditions) if err != nil { return err } } for _, sfi := range t.SiafundInputs { err := tp.checkUnlockConditions(sfi.UnlockConditions) if err != nil { return err } } // Check that all arbitrary data is prefixed using the recognized set of // prefixes. The allowed prefixes include a 'NonSia' prefix for truly // arbitrary data. Blocking all other prefixes allows arbitrary data to be // used to orchestrate more complicated soft forks in the future without // putting older nodes at risk of violating the new rules. var prefix types.Specifier for _, arb := range t.ArbitraryData { // Check for a whilelisted prefix. copy(prefix[:], arb) if prefix == modules.PrefixHostAnnouncement || prefix == modules.PrefixNonSia { continue } return modules.ErrInvalidArbPrefix } return nil } // IsStandardTransactionSet checks that all transacitons of a set follow the // IsStandard guidelines, and that the set as a whole follows the guidelines as // well. func (tp *TransactionPool) IsStandardTransactionSet(ts []types.Transaction) error { // Check that the set is a reasonable size. totalSize := 0 for i := range ts { totalSize += len(encoding.Marshal(ts[i])) if totalSize > modules.TransactionSetSizeLimit { return modules.ErrLargeTransactionSet } } // Check that each transaction is acceptable. for i := range ts { err := tp.IsStandardTransaction(ts[i]) if err != nil { return err } } return nil } Sia-1.0.4/modules/transactionpool/standard_test.go000066400000000000000000000027171302033070500223050ustar00rootroot00000000000000package transactionpool import ( "crypto/rand" "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // TestIntegrationLargeTransactions tries to add a large transaction to the // transaction pool. func TestIntegrationLargeTransactions(t *testing.T) { if testing.Short() { t.SkipNow() } tpt, err := createTpoolTester("TestIntegrationLargeTransaction") if err != nil { t.Fatal(err) } defer tpt.Close() // Create a large transaction and try to get it accepted. arbData := make([]byte, modules.TransactionSizeLimit) copy(arbData, modules.PrefixNonSia[:]) _, err = rand.Read(arbData[100:116]) // prevents collisions with other transacitons in the loop. if err != nil { t.Fatal(err) } txn := types.Transaction{ArbitraryData: [][]byte{arbData}} err = tpt.tpool.AcceptTransactionSet([]types.Transaction{txn}) if err != modules.ErrLargeTransaction { t.Fatal(err) } // Create a large transaction set and try to get it accepted. var tset []types.Transaction for i := 0; i <= modules.TransactionSetSizeLimit/10e3; i++ { arbData := make([]byte, 10e3) copy(arbData, modules.PrefixNonSia[:]) _, err = rand.Read(arbData[100:116]) // prevents collisions with other transacitons in the loop. if err != nil { t.Fatal(err) } txn := types.Transaction{ArbitraryData: [][]byte{arbData}} tset = append(tset, txn) } err = tpt.tpool.AcceptTransactionSet(tset) if err != modules.ErrLargeTransactionSet { t.Fatal(err) } } Sia-1.0.4/modules/transactionpool/subscribe.go000066400000000000000000000036751302033070500214330ustar00rootroot00000000000000package transactionpool import ( "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // updateSubscribersTransactions sends a new transaction pool update to all // subscribers. func (tp *TransactionPool) updateSubscribersTransactions() { var txns []types.Transaction var cc modules.ConsensusChange for _, tSet := range tp.transactionSets { txns = append(txns, tSet...) } for _, tSetDiff := range tp.transactionSetDiffs { cc = cc.Append(tSetDiff) } for _, subscriber := range tp.subscribers { subscriber.ReceiveUpdatedUnconfirmedTransactions(txns, cc) } } // TransactionPoolSubscribe adds a subscriber to the transaction pool. // Subscribers will receive the full transaction set every time there is a // significant change to the transaction pool. func (tp *TransactionPool) TransactionPoolSubscribe(subscriber modules.TransactionPoolSubscriber) { tp.mu.Lock() defer tp.mu.Unlock() // Add the subscriber to the subscriber list. tp.subscribers = append(tp.subscribers, subscriber) // Send the new subscriber the transaction pool set. var txns []types.Transaction for _, tSet := range tp.transactionSets { txns = append(txns, tSet...) } var cc modules.ConsensusChange for _, tSetDiff := range tp.transactionSetDiffs { cc = cc.Append(tSetDiff) } subscriber.ReceiveUpdatedUnconfirmedTransactions(txns, cc) } // Unsubscribe removes a subscriber from the transaction pool. If the // subscriber is not in tp.subscribers, Unsubscribe does nothing. If the // subscriber occurs more than once in tp.subscribers, only the earliest // occurrence is removed (unsubscription fails). func (tp *TransactionPool) Unsubscribe(subscriber modules.TransactionPoolSubscriber) { tp.mu.Lock() defer tp.mu.Unlock() // Search for and remove subscriber from list of subscribers. for i := range tp.subscribers { if tp.subscribers[i] == subscriber { tp.subscribers = append(tp.subscribers[0:i], tp.subscribers[i+1:]...) break } } } Sia-1.0.4/modules/transactionpool/subscribe_test.go000066400000000000000000000043771302033070500224720ustar00rootroot00000000000000package transactionpool import ( "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "testing" ) // mockSubscriber receives transactions from the transaction pool it is // subscribed to, retaining them in the order they were received. type mockSubscriber struct { txns []types.Transaction } // ReceiveUpdatedUnconfirmedTransactions receives transactinos from the // transaction pool and stores them in the order they were received. // This method allows *mockSubscriber to satisfy the // modules.TransactionPoolSubscriber interface. func (ms *mockSubscriber) ReceiveUpdatedUnconfirmedTransactions(txns []types.Transaction, _ modules.ConsensusChange) { ms.txns = append(ms.txns, txns...) } // TestSubscription checks that calling Unsubscribe on a mockSubscriber // shortens the list of subscribers to the transaction pool by 1 (doesn't // actually check that the mockSubscriber was the one unsubscribed). func TestSubscription(t *testing.T) { if testing.Short() { t.Skip() } tpt, err := createTpoolTester("TestUnsubscribe") if err != nil { t.Fatal(err) } defer tpt.Close() // Check the transaction pool is empty when initialized. if len(tpt.tpool.transactionSets) != 0 { t.Fatal("transaction pool is not empty") } // Create a mock subscriber and subscribe it to the transaction pool. ms := mockSubscriber{} tpt.tpool.TransactionPoolSubscribe(&ms) if len(ms.txns) != 0 { t.Fatalf("mock subscriber has received %v transactions; shouldn't have received any yet", len(ms.txns)) } // Create a valid transaction set and check that the mock subscriber's // transaction list is updated. _, err = tpt.wallet.SendSiacoins(types.NewCurrency64(100), types.UnlockHash{}) if err != nil { t.Fatal(err) } if len(tpt.tpool.transactionSets) != 1 { t.Error("sending coins didn't increase the transaction sets by 1") } numTxns := 0 for _, txnSet := range tpt.tpool.transactionSets { numTxns += len(txnSet) } if len(ms.txns) != numTxns { t.Errorf("mock subscriber should've received %v transactions; received %v instead", numTxns, len(ms.txns)) } numSubscribers := len(tpt.tpool.subscribers) tpt.tpool.Unsubscribe(&ms) if len(tpt.tpool.subscribers) != numSubscribers-1 { t.Error("transaction pool failed to unsubscribe mock subscriber") } } Sia-1.0.4/modules/transactionpool/transactionpool.go000066400000000000000000000115721302033070500226640ustar00rootroot00000000000000package transactionpool import ( "errors" "github.com/NebulousLabs/demotemutex" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/types" ) const ( dbFilename = "transactionpool.db" ) var ( dbMetadata = persist.Metadata{ Header: "Sia Transaction Pool DB", Version: "0.6.0", } errNilCS = errors.New("transaction pool cannot initialize with a nil consensus set") errNilGateway = errors.New("transaction pool cannot initialize with a nil gateway") ) type ( // ObjectIDs are the IDs of objects such as siacoin outputs and file // contracts, and are used to see if there are conflicts or overlaps within // the transaction pool. A TransactionSetID is the hash of a transaction // set. ObjectID crypto.Hash TransactionSetID crypto.Hash // The TransactionPool tracks incoming transactions, accepting them or // rejecting them based on internal criteria such as fees and unconfirmed // double spends. TransactionPool struct { // Dependencies of the transaction pool. consensusSet modules.ConsensusSet gateway modules.Gateway // To prevent double spends in the unconfirmed transaction set, the // transaction pool keeps a list of all objects that have either been // created or consumed by the current unconfirmed transaction pool. All // transactions with overlaps are rejected. This model is // over-aggressive - one transaction set may create an object that // another transaction set spends. This is done to minimize the // computation and memory load on the transaction pool. Dependent // transactions should be lumped into a single transaction set. // // transactionSetDiffs map form a transaction set id to the set of // diffs that resulted from the transaction set. knownObjects map[ObjectID]TransactionSetID transactionSets map[TransactionSetID][]types.Transaction transactionSetDiffs map[TransactionSetID]modules.ConsensusChange transactionListSize int // TODO: Write a consistency check comparing transactionSets, // transactionSetDiffs. // // TODO: Write a consistency check making sure that all unconfirmedIDs // point to the right place, and that all UnconfirmedIDs are accounted for. // The consensus change index tracks how many consensus changes have // been sent to the transaction pool. When a new subscriber joins the // transaction pool, all prior consensus changes are sent to the new // subscriber. subscribers []modules.TransactionPoolSubscriber // Utilities. db *persist.BoltDatabase mu demotemutex.DemoteMutex persistDir string } ) // New creates a transaction pool that is ready to receive transactions. func New(cs modules.ConsensusSet, g modules.Gateway, persistDir string) (*TransactionPool, error) { // Check that the input modules are non-nil. if cs == nil { return nil, errNilCS } if g == nil { return nil, errNilGateway } // Initialize a transaction pool. tp := &TransactionPool{ consensusSet: cs, gateway: g, knownObjects: make(map[ObjectID]TransactionSetID), transactionSets: make(map[TransactionSetID][]types.Transaction), transactionSetDiffs: make(map[TransactionSetID]modules.ConsensusChange), persistDir: persistDir, } // Open the tpool database. err := tp.initPersist() if err != nil { return nil, err } // Register RPCs g.RegisterRPC("RelayTransactionSet", tp.relayTransactionSet) return tp, nil } func (tp *TransactionPool) Close() error { tp.gateway.UnregisterRPC("RelayTransactionSet") tp.consensusSet.Unsubscribe(tp) return tp.db.Close() } // FeeEstimation returns an estimation for what fee should be applied to // transactions. func (tp *TransactionPool) FeeEstimation() (min, max types.Currency) { // TODO: The fee estimation tool should look at the recent blocks and use // them to guage what sort of fee should be required, as opposed to just // guessing blindly. // // TODO: The current minimum has been reduced significantly to account for // legacy renters that are not correctly adding transaction fees. The // minimum has been set to 1 siacoin per kb (or 1/1000 SC per byte), but // really should look more like 10 SC per kb. But, legacy renters are using // a much lower value, which means hosts would be incompatible if the // minimum recommended were set to 10. The value has been set to 1, which // should be okay temporarily while the renters are given time to upgrade. return types.SiacoinPrecision.Mul64(1).Div64(1e3), types.SiacoinPrecision.Mul64(5).Div64(1e3) } // TransactionList returns a list of all transactions in the transaction pool. // The transactions are provided in an order that can acceptably be put into a // block. func (tp *TransactionPool) TransactionList() []types.Transaction { var txns []types.Transaction for _, tSet := range tp.transactionSets { txns = append(txns, tSet...) } return txns } Sia-1.0.4/modules/transactionpool/transactionpool_test.go000066400000000000000000000067041302033070500237240ustar00rootroot00000000000000package transactionpool import ( "crypto/rand" "path/filepath" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/consensus" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/modules/miner" "github.com/NebulousLabs/Sia/modules/wallet" "github.com/NebulousLabs/Sia/types" ) // A tpoolTester is used during testing to initialize a transaction pool and // useful helper modules. type tpoolTester struct { cs modules.ConsensusSet gateway modules.Gateway tpool *TransactionPool miner modules.TestMiner wallet modules.Wallet walletKey crypto.TwofishKey persistDir string } // createTpoolTester returns a ready-to-use tpool tester, with all modules // initialized. func createTpoolTester(name string) (*tpoolTester, error) { // Initialize the modules. testdir := build.TempDir(modules.TransactionPoolDir, name) g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, err } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } tp, err := New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { return nil, err } w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } var key crypto.TwofishKey _, err = rand.Read(key[:]) if err != nil { return nil, err } _, err = w.Encrypt(key) if err != nil { return nil, err } err = w.Unlock(key) if err != nil { return nil, err } m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.MinerDir)) if err != nil { return nil, err } // Assemble all of the objects into a tpoolTester tpt := &tpoolTester{ cs: cs, gateway: g, tpool: tp, miner: m, wallet: w, walletKey: key, persistDir: testdir, } // Mine blocks until there is money in the wallet. for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ { b, _ := tpt.miner.FindBlock() err = tpt.cs.AcceptBlock(b) if err != nil { return nil, err } } return tpt, nil } // Close safely closes the tpoolTester, calling a panic in the event of an // error since there isn't a good way to errcheck when deferring a Close. func (tpt *tpoolTester) Close() error { errs := []error{ tpt.cs.Close(), tpt.gateway.Close(), tpt.tpool.Close(), tpt.miner.Close(), // TODO: implement modules.Wallet.Close() // tpt.wallet.Close() } if err := build.JoinErrors(errs, "; "); err != nil { panic(err) } return nil } // TestIntegrationNewNilInputs tries to trigger a panic with nil inputs. func TestIntegrationNewNilInputs(t *testing.T) { // Create a gateway and consensus set. testdir := build.TempDir(modules.TransactionPoolDir, "TestNewNilInputs") g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { t.Fatal(err) } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { t.Fatal(err) } tpDir := filepath.Join(testdir, modules.TransactionPoolDir) // Try all combinations of nil inputs. _, err = New(nil, nil, tpDir) if err == nil { t.Error(err) } _, err = New(nil, g, tpDir) if err != errNilCS { t.Error(err) } _, err = New(cs, nil, tpDir) if err != errNilGateway { t.Error(err) } _, err = New(cs, g, tpDir) if err != nil { t.Error(err) } } Sia-1.0.4/modules/transactionpool/update.go000066400000000000000000000106101302033070500207170ustar00rootroot00000000000000package transactionpool import ( "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/NebulousLabs/bolt" ) // purge removes all transactions from the transaction pool. func (tp *TransactionPool) purge() { tp.knownObjects = make(map[ObjectID]TransactionSetID) tp.transactionSets = make(map[TransactionSetID][]types.Transaction) tp.transactionSetDiffs = make(map[TransactionSetID]modules.ConsensusChange) tp.transactionListSize = 0 } // ProcessConsensusChange gets called to inform the transaction pool of changes // to the consensus set. func (tp *TransactionPool) ProcessConsensusChange(cc modules.ConsensusChange) { tp.mu.Lock() // Update the database of confirmed transactions. err := tp.db.Update(func(tx *bolt.Tx) error { for _, block := range cc.RevertedBlocks { for _, txn := range block.Transactions { err := tp.deleteTransaction(tx, txn.ID()) if err != nil { return err } } } for _, block := range cc.AppliedBlocks { for _, txn := range block.Transactions { err := tp.addTransaction(tx, txn.ID()) if err != nil { return err } } } return tp.putRecentConsensusChange(tx, cc.ID) }) if err != nil { // TODO: Handle error } // Scan the applied blocks for transactions that got accepted. This will // help to determine which transactions to remove from the transaction // pool. Having this list enables both efficiency improvements and helps to // clean out transactions with no dependencies, such as arbitrary data // transactions from the host. txids := make(map[types.TransactionID]struct{}) for _, block := range cc.AppliedBlocks { for _, txn := range block.Transactions { txids[txn.ID()] = struct{}{} } } // TODO: Right now, transactions that were reverted to not get saved and // retried, because some transactions such as storage proofs might be // illegal, and there's no good way to preserve dependencies when illegal // transactions are suddenly involved. // // One potential solution is to have modules manually do resubmission if // something goes wrong. Another is to have the transaction pool remember // recent transaction sets on the off chance that they become valid again // due to a reorg. // // Another option is to scan through the blocks transactions one at a time // check if they are valid. If so, lump them in a set with the next guy. // When they stop being valid, you've found a guy to throw away. It's n^2 // in the number of transactions in the block. // Save all of the current unconfirmed transaction sets into a list. var unconfirmedSets [][]types.Transaction for _, tSet := range tp.transactionSets { // Compile a new transaction set the removes all transactions duplicated // in the block. Though mostly handled by the dependency manager in the // transaction pool, this should both improve efficiency and will strip // out duplicate transactions with no dependencies (arbitrary data only // transactions) var newTSet []types.Transaction for _, txn := range tSet { _, exists := txids[txn.ID()] if !exists { newTSet = append(newTSet, txn) } } unconfirmedSets = append(unconfirmedSets, newTSet) } // Purge the transaction pool. Some of the transactions sets may be invalid // after the consensus change. tp.purge() // Add all of the unconfirmed transaction sets back to the transaction // pool. The ones that are invalid will throw an error and will not be // re-added. // // Accepting a transaction set requires locking the consensus set (to check // validity). But, ProcessConsensusChange is only called when the consensus // set is already locked, causing a deadlock problem. Therefore, // transactions are readded to the pool in a goroutine, so that this // function can finish and consensus can unlock. The tpool lock is held // however until the goroutine completes. // // Which means that no other modules can require a tpool lock when // processing consensus changes. Overall, the locking is pretty fragile and // more rules need to be put in place. for _, set := range unconfirmedSets { tp.acceptTransactionSet(set) // Error is not checked. } // Inform subscribers that an update has executed. tp.mu.Demote() tp.updateSubscribersTransactions() tp.mu.DemotedUnlock() } // PurgeTransactionPool deletes all transactions from the transaction pool. func (tp *TransactionPool) PurgeTransactionPool() { tp.mu.Lock() tp.purge() tp.mu.Unlock() } Sia-1.0.4/modules/transactionpool/update_test.go000066400000000000000000000020531302033070500217600ustar00rootroot00000000000000package transactionpool import ( "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // TestArbDataOnly tries submitting a transaction with only arbitrary data to // the transaction pool. Then a block is mined, putting the transaction on the // blockchain. The arb data transaction should no longer be in the transaction // pool. func TestArbDataOnly(t *testing.T) { if testing.Short() { t.SkipNow() } tpt, err := createTpoolTester("TestArbDataOnly") if err != nil { t.Fatal(err) } defer tpt.Close() txn := types.Transaction{ ArbitraryData: [][]byte{ append(modules.PrefixNonSia[:], []byte("arb-data")...), }, } err = tpt.tpool.AcceptTransactionSet([]types.Transaction{txn}) if err != nil { t.Fatal(err) } if len(tpt.tpool.TransactionList()) != 1 { t.Error("expecting to see a transaction in the transaction pool") } _, err = tpt.miner.AddBlock() if err != nil { t.Fatal(err) } if len(tpt.tpool.TransactionList()) != 0 { t.Error("transaction was not cleared from the transaction pool") } } Sia-1.0.4/modules/transactionpool_test.go000066400000000000000000000063221302033070500205010ustar00rootroot00000000000000package modules import ( "testing" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/types" ) // TestConsensusConflict checks that the consensus conflict type is correctly // assembling consensus conflict errors. func TestConsensusConflict(t *testing.T) { t.Parallel() ncc := NewConsensusConflict("problem") if ncc.Error() != "consensus conflict: problem" { t.Error("wrong error message being reported in a consensus conflict") } err := func() error { return ncc }() if err.Error() != "consensus conflict: problem" { t.Error("wrong error message being reported in a consensus conflict") } if _, ok := err.(ConsensusConflict); !ok { t.Error("error is not maintaining consensus conflict type") } } // TestCalculateFee checks that the CalculateFee function is correctly tallying // the number of fees in a transaction set. func TestCalculateFee(t *testing.T) { t.Parallel() // Try calculating the fees on a nil transaction set. if CalculateFee(nil).Cmp(types.ZeroCurrency) != 0 { t.Error("CalculateFee is incorrectly handling nil input") } // Try a single transaction with no fees. txnSet := []types.Transaction{{}} if CalculateFee(txnSet).Cmp(types.ZeroCurrency) != 0 { t.Error("CalculateFee is not correctly calculating the fees on an empty transaction set") } // Try a non-empty transaction. txnSet = []types.Transaction{{ SiacoinOutputs: []types.SiacoinOutput{{ Value: types.NewCurrency64(253e9), }}, }} if CalculateFee(txnSet).Cmp(types.ZeroCurrency) != 0 { t.Error("CalculateFee is not correctly calculating the fees on a non-empty transaction set") } // Try a transaction set with a single miner fee. baseFee := types.NewCurrency64(12e3) txnSet = []types.Transaction{{ MinerFees: []types.Currency{ baseFee, }, }} setLen := uint64(len(encoding.Marshal(txnSet))) expectedFee := baseFee.Div64(setLen) if CalculateFee(txnSet).Cmp(expectedFee) != 0 { t.Error("CalculateFee doesn't seem to be calculating the correct transaction fee") } // Try the transaction set when there is more data. txnSet[0].FileContracts = append(txnSet[0].FileContracts, types.FileContract{ FileSize: 10e3, }) newSetLen := uint64(len(encoding.Marshal(txnSet))) if newSetLen <= setLen { t.Fatal("transaction set did not grow after adding a file contract") } newExpectedFee := baseFee.Div64(newSetLen) if newExpectedFee.Cmp(expectedFee) >= 0 { t.Error("the new expected fee should go down as the txn size increases") } if CalculateFee(txnSet).Cmp(newExpectedFee) != 0 { t.Error("the new expected fee does not match the new actual fee") } // Try a transaction set with multiple transactions and multiple fees per // transaction. fee1 := types.NewCurrency64(1e6) fee2 := types.NewCurrency64(2e6) fee3 := types.NewCurrency64(3e6) fee4 := types.NewCurrency64(4e6) txnSet = []types.Transaction{ { MinerFees: []types.Currency{ fee1, fee2, }, }, { MinerFees: []types.Currency{ fee3, fee4, }, }, } currencyLen := types.NewCurrency64(uint64(len(encoding.Marshal(txnSet)))) multiExpectedFee := fee1.Add(fee2).Add(fee3).Add(fee4).Div(currencyLen) if CalculateFee(txnSet).Cmp(multiExpectedFee) != 0 { t.Error("got the wrong fee for a multi transaction set") } } Sia-1.0.4/modules/wallet.go000066400000000000000000000425031302033070500155140ustar00rootroot00000000000000package modules import ( "bytes" "errors" "github.com/NebulousLabs/entropy-mnemonics" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/types" ) const ( // WalletDir is the directory that contains the wallet persistence. WalletDir = "wallet" // SeedChecksumSize is the number of bytes that are used to checksum // addresses to prevent accidental spending. SeedChecksumSize = 6 // PublicKeysPerSeed define the number of public keys that get pregenerated // for a seed at startup when searching for balances in the blockchain. PublicKeysPerSeed = 2500 // WalletSeedPreloadDepth is the number of addresses that get automatically // loaded by the wallet at startup. WalletSeedPreloadDepth = 25 ) var ( // ErrBadEncryptionKey is returned if the incorrect encryption key to a // file is provided. ErrBadEncryptionKey = errors.New("provided encryption key is incorrect") // ErrLowBalance is returned if the wallet does not have enough funds to // complete the desired action. ErrLowBalance = errors.New("insufficient balance") // ErrIncompleteTransactions is returned if the wallet has incomplete // transactions being built that are using all of the current outputs, and // therefore the wallet is unable to spend money despite it not technically // being 'unconfirmed' yet. ErrIncompleteTransactions = errors.New("wallet has coins spent in incomplete transactions - not enough remaining coins") // ErrLockedWallet is returned when an action cannot be performed due to // the wallet being locked. ErrLockedWallet = errors.New("wallet must be unlocked before it can be used") ) type ( // Seed is cryptographic entropy that is used to derive spendable wallet // addresses. Seed [crypto.EntropySize]byte // WalletTransactionID is a unique identifier for a wallet transaction. WalletTransactionID crypto.Hash // A ProcessedInput represents funding to a transaction. The input is // coming from an address and going to the outputs. The fund types are // 'SiacoinInput', 'SiafundInput'. ProcessedInput struct { FundType types.Specifier `json:"fundtype"` WalletAddress bool `json:"walletaddress"` RelatedAddress types.UnlockHash `json:"relatedaddress"` Value types.Currency `json:"value"` } // A ProcessedOutput is a siacoin output that appears in a transaction. // Some outputs mature immediately, some are delayed, and some may never // mature at all (in the event of storage proofs). // // Fund type can either be 'SiacoinOutput', 'SiafundOutput', 'ClaimOutput', // 'MinerPayout', or 'MinerFee'. All outputs except the miner fee create // outputs accessible to an address. Miner fees are not spendable, and // instead contribute to the block subsidy. // // MaturityHeight indicates at what block height the output becomes // available. SiacoinInputs and SiafundInputs become available immediately. // ClaimInputs and MinerPayouts become available after 144 confirmations. ProcessedOutput struct { FundType types.Specifier `json:"fundtype"` MaturityHeight types.BlockHeight `json:"maturityheight"` WalletAddress bool `json:"walletaddress"` RelatedAddress types.UnlockHash `json:"relatedaddress"` Value types.Currency `json:"value"` } // A ProcessedTransaction is a transaction that has been processed into // explicit inputs and outputs and tagged with some header data such as // confirmation height + timestamp. // // Because of the block subsidy, a block is considered as a transaction. // Since there is technically no transaction id for the block subsidy, the // block id is used instead. ProcessedTransaction struct { Transaction types.Transaction `json:"transaction"` TransactionID types.TransactionID `json:"transactionid"` ConfirmationHeight types.BlockHeight `json:"confirmationheight"` ConfirmationTimestamp types.Timestamp `json:"confirmationtimestamp"` Inputs []ProcessedInput `json:"inputs"` Outputs []ProcessedOutput `json:"outputs"` } // TransactionBuilder is used to construct custom transactions. A transaction // builder is initialized via 'RegisterTransaction' and then can be modified by // adding funds or other fields. The transaction is completed by calling // 'Sign', which will sign all inputs added via the 'FundSiacoins' or // 'FundSiafunds' call. All modifications are additive. // // Parents of the transaction are kept in the transaction builder. A parent is // any unconfirmed transaction that is required for the child to be valid. // // Transaction builders are not thread safe. TransactionBuilder interface { // FundSiacoins will add a siacoin input of exactly 'amount' to the // transaction. A parent transaction may be needed to achieve an input // with the correct value. The siacoin input will not be signed until // 'Sign' is called on the transaction builder. The expectation is that // the transaction will be completed and broadcast within a few hours. // Longer risks double-spends, as the wallet will assume that the // transaction failed. FundSiacoins(amount types.Currency) error // FundSiafunds will add a siafund input of exactly 'amount' to the // transaction. A parent transaction may be needed to achieve an input // with the correct value. The siafund input will not be signed until // 'Sign' is called on the transaction builder. Any siacoins that are // released by spending the siafund outputs will be sent to another // address owned by the wallet. The expectation is that the transaction // will be completed and broadcast within a few hours. Longer risks // double-spends, because the wallet will assume the transaction // failed. FundSiafunds(amount types.Currency) error // AddParents adds a set of parents to the transaction. AddParents([]types.Transaction) // AddMinerFee adds a miner fee to the transaction, returning the index // of the miner fee within the transaction. AddMinerFee(fee types.Currency) uint64 // AddSiacoinInput adds a siacoin input to the transaction, returning // the index of the siacoin input within the transaction. When 'Sign' // gets called, this input will be left unsigned. AddSiacoinInput(types.SiacoinInput) uint64 // AddSiacoinOutput adds a siacoin output to the transaction, returning // the index of the siacoin output within the transaction. AddSiacoinOutput(types.SiacoinOutput) uint64 // AddFileContract adds a file contract to the transaction, returning // the index of the file contract within the transaction. AddFileContract(types.FileContract) uint64 // AddFileContractRevision adds a file contract revision to the // transaction, returning the index of the file contract revision // within the transaction. When 'Sign' gets called, this revision will // be left unsigned. AddFileContractRevision(types.FileContractRevision) uint64 // AddStorageProof adds a storage proof to the transaction, returning // the index of the storage proof within the transaction. AddStorageProof(types.StorageProof) uint64 // AddSiafundInput adds a siafund input to the transaction, returning // the index of the siafund input within the transaction. When 'Sign' // is called, this input will be left unsigned. AddSiafundInput(types.SiafundInput) uint64 // AddSiafundOutput adds a siafund output to the transaction, returning // the index of the siafund output within the transaction. AddSiafundOutput(types.SiafundOutput) uint64 // AddArbitraryData adds arbitrary data to the transaction, returning // the index of the data within the transaction. AddArbitraryData(arb []byte) uint64 // AddTransactionSignature adds a transaction signature to the // transaction, returning the index of the signature within the // transaction. The signature should already be valid, and shouldn't // sign any of the inputs that were added by calling 'FundSiacoins' or // 'FundSiafunds'. AddTransactionSignature(types.TransactionSignature) uint64 // Sign will sign any inputs added by 'FundSiacoins' or 'FundSiafunds' // and return a transaction set that contains all parents prepended to // the transaction. If more fields need to be added, a new transaction // builder will need to be created. // // If the whole transaction flag is set to true, then the whole // transaction flag will be set in the covered fields object. If the // whole transaction flag is set to false, then the covered fields // object will cover all fields that have already been added to the // transaction, but will also leave room for more fields to be added. // // An error will be returned if there are multiple calls to 'Sign', // sometimes even if the first call to Sign has failed. Sign should // only ever be called once, and if the first signing fails, the // transaction should be dropped. Sign(wholeTransaction bool) ([]types.Transaction, error) // View returns the incomplete transaction along with all of its // parents. View() (txn types.Transaction, parents []types.Transaction) // ViewAdded returns all of the siacoin inputs, siafund inputs, and // parent transactions that have been automatically added by the // builder. Items are returned by index. ViewAdded() (newParents, siacoinInputs, siafundInputs, transactionSignatures []int) // Drop indicates that a transaction is no longer useful and will not be // broadcast, and that all of the outputs can be reclaimed. 'Drop' // should only be used before signatures are added. Drop() } // EncryptionManager can encrypt, lock, unlock, and indicate the current // status of the EncryptionManager. EncryptionManager interface { // Encrypt will encrypt the wallet using the input key. Upon // encryption, a primary seed will be created for the wallet (no seed // exists prior to this point). If the key is blank, then the hash of // the seed that is generated will be used as the key. // // Encrypt can only be called once throughout the life of the wallet // and will return an error on subsequent calls (even after restarting // the wallet). To reset the wallet, the wallet files must be moved to // a different directory or deleted. Encrypt(masterKey crypto.TwofishKey) (Seed, error) // Encrypted returns whether or not the wallet has been encrypted yet. // After being encrypted for the first time, the wallet can only be // unlocked using the encryption password. Encrypted() bool // Lock deletes all keys in memory and prevents the wallet from being // used to spend coins or extract keys until 'Unlock' is called. Lock() error // Unlock must be called before the wallet is usable. All wallets and // wallet seeds are encrypted by default, and the wallet will not know // which addresses to watch for on the blockchain until unlock has been // called. // // All items in the wallet are encrypted using different keys which are // derived from the master key. Unlock(masterKey crypto.TwofishKey) error // Unlocked returns true if the wallet is currently unlocked, false // otherwise. Unlocked() bool } // KeyManager manages wallet keys, including the use of seeds, creating and // loading backups, and providing a layer of compatibility for older wallet // files. KeyManager interface { // AllAddresses returns all addresses that the wallet is able to spend // from, including unseeded addresses. Addresses are returned sorted in // byte-order. AllAddresses() []types.UnlockHash // AllSeeds returns all of the seeds that are being tracked by the // wallet, including the primary seed. Only the primary seed is used to // generate new addresses, but the wallet can spend funds sent to // public keys generated by any of the seeds returned. AllSeeds() ([]Seed, error) // PrimarySeed returns the current primary seed of the wallet, // unencrypted, with an int indicating how many addresses have been // consumed. PrimarySeed() (Seed, uint64, error) // NextAddress returns a new coin addresses generated from the // primary seed. NextAddress() (types.UnlockConditions, error) // CreateBackup will create a backup of the wallet at the provided // filepath. The backup will have all seeds and keys. CreateBackup(string) error // LoadBackup will load a backup of the wallet from the provided // address. The backup wallet will be added as an auxiliary seed, not // as a primary seed. // LoadBackup(masterKey, backupMasterKey crypto.TwofishKey, string) error // Load033xWallet will load a version 0.3.3.x wallet from disk and add all of // the keys in the wallet as unseeded keys. Load033xWallet(crypto.TwofishKey, string) error // LoadSeed will recreate a wallet file using the recovery phrase. // LoadSeed only needs to be called if the original seed file or // encryption password was lost. The master key is used to encrypt the // recovery seed before saving it to disk. LoadSeed(crypto.TwofishKey, Seed) error // LoadSiagKeys will take a set of filepaths that point to a siag key // and will have the siag keys loaded into the wallet so that they will // become spendable. LoadSiagKeys(crypto.TwofishKey, []string) error } // Wallet stores and manages siacoins and siafunds. The wallet file is // encrypted using a user-specified password. Common addresses are all // derived from a single address seed. Wallet interface { EncryptionManager KeyManager // Close permits clean shutdown during testing and serving. Close() error // ConfirmedBalance returns the confirmed balance of the wallet, minus // any outgoing transactions. ConfirmedBalance will include unconfirmed // refund transactions. ConfirmedBalance() (siacoinBalance types.Currency, siafundBalance types.Currency, siacoinClaimBalance types.Currency) // UnconfirmedBalance returns the unconfirmed balance of the wallet. // Outgoing funds and incoming funds are reported separately. Refund // outputs are included, meaning that sending a single coin to // someone could result in 'outgoing: 12, incoming: 11'. Siafunds are // not considered in the unconfirmed balance. UnconfirmedBalance() (outgoingSiacoins types.Currency, incomingSiacoins types.Currency) // AddressTransactions returns all of the transactions that are related // to a given address. AddressTransactions(types.UnlockHash) []ProcessedTransaction // AddressUnconfirmedHistory returns all of the unconfirmed // transactions related to a given address. AddressUnconfirmedTransactions(types.UnlockHash) []ProcessedTransaction // Transaction returns the transaction with the given id. The bool // indicates whether the transaction is in the wallet database. The // wallet only stores transactions that are related to the wallet. Transaction(types.TransactionID) (ProcessedTransaction, bool) // Transactions returns all of the transactions that were confirmed at // heights [startHeight, endHeight]. Unconfirmed transactions are not // included. Transactions(startHeight types.BlockHeight, endHeight types.BlockHeight) ([]ProcessedTransaction, error) // UnconfirmedTransactions returns all unconfirmed transactions // relative to the wallet. UnconfirmedTransactions() []ProcessedTransaction // RegisterTransaction takes a transaction and its parents and returns // a TransactionBuilder which can be used to expand the transaction. RegisterTransaction(t types.Transaction, parents []types.Transaction) TransactionBuilder // StartTransaction is a convenience method that calls // RegisterTransaction(types.Transaction{}, nil) StartTransaction() TransactionBuilder // SendSiacoins is a tool for sending siacoins from the wallet to an // address. Sending money usually results in multiple transactions. The // transactions are automatically given to the transaction pool, and // are also returned to the caller. SendSiacoins(amount types.Currency, dest types.UnlockHash) ([]types.Transaction, error) // SendSiafunds is a tool for sending siafunds from the wallet to an // address. Sending money usually results in multiple transactions. The // transactions are automatically given to the transaction pool, and // are also returned to the caller. SendSiafunds(amount types.Currency, dest types.UnlockHash) ([]types.Transaction, error) } ) // CalculateWalletTransactionID is a helper function for determining the id of // a wallet transaction. func CalculateWalletTransactionID(tid types.TransactionID, oid types.OutputID) WalletTransactionID { return WalletTransactionID(crypto.HashAll(tid, oid)) } // SeedToString converts a wallet seed to a human friendly string. func SeedToString(seed Seed, did mnemonics.DictionaryID) (string, error) { fullChecksum := crypto.HashObject(seed) checksumSeed := append(seed[:], fullChecksum[:SeedChecksumSize]...) phrase, err := mnemonics.ToPhrase(checksumSeed, did) if err != nil { return "", err } return phrase.String(), nil } // StringToSeed converts a string to a wallet seed. func StringToSeed(str string, did mnemonics.DictionaryID) (Seed, error) { // Decode the string into the checksummed byte slice. checksumSeedBytes, err := mnemonics.FromString(str, did) if err != nil { return Seed{}, err } // Copy the seed from the checksummed slice. var seed Seed copy(seed[:], checksumSeedBytes) fullChecksum := crypto.HashObject(seed) if len(checksumSeedBytes) != crypto.EntropySize+SeedChecksumSize || !bytes.Equal(fullChecksum[:SeedChecksumSize], checksumSeedBytes[crypto.EntropySize:]) { return Seed{}, errors.New("seed failed checksum verification") } return seed, nil } Sia-1.0.4/modules/wallet/000077500000000000000000000000001302033070500151615ustar00rootroot00000000000000Sia-1.0.4/modules/wallet/encrypt.go000066400000000000000000000167011302033070500172010ustar00rootroot00000000000000package wallet import ( "bytes" "crypto/rand" "errors" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( errAlreadyUnlocked = errors.New("wallet has already been unlocked") errReencrypt = errors.New("wallet is already encrypted, cannot encrypt again") errUnencryptedWallet = errors.New("wallet has not been encrypted yet") unlockModifier = types.Specifier{'u', 'n', 'l', 'o', 'c', 'k'} ) // uidEncryptionKey creates an encryption key that is used to decrypt a // specific key file. func uidEncryptionKey(masterKey crypto.TwofishKey, uid UniqueID) crypto.TwofishKey { return crypto.TwofishKey(crypto.HashAll(masterKey, uid)) } // checkMasterKey verifies that the master key is correct. func (w *Wallet) checkMasterKey(masterKey crypto.TwofishKey) error { uk := uidEncryptionKey(masterKey, w.persist.UID) verification, err := uk.DecryptBytes(w.persist.EncryptionVerification) if err != nil { // Most of the time, the failure is an authentication failure. return modules.ErrBadEncryptionKey } expected := make([]byte, encryptionVerificationLen) if !bytes.Equal(expected, verification) { return modules.ErrBadEncryptionKey } return nil } // initEncryption checks that the provided encryption key is the valid // encryption key for the wallet. If encryption has not yet been established // for the wallet, an encryption key is created. func (w *Wallet) initEncryption(masterKey crypto.TwofishKey) (modules.Seed, error) { // Check if the wallet encryption key has already been set. if len(w.persist.EncryptionVerification) != 0 { return modules.Seed{}, errReencrypt } // Create a random seed and use it to generate the seed file for the // wallet. var seed modules.Seed _, err := rand.Read(seed[:]) if err != nil { return modules.Seed{}, err } // If the input key is blank, use the seed to create the master key. // Otherwise, use the input key. if masterKey == (crypto.TwofishKey{}) { masterKey = crypto.TwofishKey(crypto.HashObject(seed)) } err = w.createSeed(masterKey, seed) if err != nil { return modules.Seed{}, err } // Establish the encryption verification using the masterKey. After this // point, the wallet is encrypted. uk := uidEncryptionKey(masterKey, w.persist.UID) encryptionBase := make([]byte, encryptionVerificationLen) w.persist.EncryptionVerification, err = uk.EncryptBytes(encryptionBase) if err != nil { return modules.Seed{}, err } err = w.saveSettings() if err != nil { return modules.Seed{}, err } return seed, nil } // managedUnlock loads all of the encrypted file structures into wallet memory. Even // after loading, the structures are kept encrypted, but some data such as // addresses are decrypted so that the wallet knows what to track. func (w *Wallet) managedUnlock(masterKey crypto.TwofishKey) error { var subscribed bool err := func() error { w.mu.Lock() defer w.mu.Unlock() subscribed = w.subscribed // Wallet should only be unlocked once. if w.unlocked { return errAlreadyUnlocked } // Check if the wallet encryption key has already been set. if len(w.persist.EncryptionVerification) == 0 { return errUnencryptedWallet } // Initialize the encryption of the wallet. err := w.checkMasterKey(masterKey) if err != nil { return err } // Load the wallet seed that is used to generate new addresses. err = w.initPrimarySeed(masterKey) if err != nil { return err } // Load all wallet seeds that are not used to generate new addresses. err = w.initAuxiliarySeeds(masterKey) if err != nil { return err } // Load all keys that were not generated by a seed. return w.initUnseededKeys(masterKey) }() if err != nil { return err } // Subscribe to the consensus set if this is the first unlock for the // wallet object. if !subscribed { // During rescan, print height every 3 seconds. if build.Release != "testing" { go func() { println("Rescanning consensus set...") for range time.Tick(time.Second * 3) { w.mu.RLock() height := w.consensusSetHeight done := w.subscribed w.mu.RUnlock() if done { println("\nDone!") break } print("\rScanned to height ", height, "...") } }() } err = w.cs.ConsensusSetSubscribe(w, modules.ConsensusChangeBeginning) if err != nil { return errors.New("wallet subscription failed: " + err.Error()) } w.tpool.TransactionPoolSubscribe(w) w.mu.Lock() w.subscribed = true w.mu.Unlock() } w.mu.Lock() w.unlocked = true w.mu.Unlock() return nil } // wipeSecrets erases all of the seeds and secret keys in the wallet. func (w *Wallet) wipeSecrets() { // 'for i := range' must be used to prevent copies of secret data from // being made. for i := range w.keys { for j := range w.keys[i].SecretKeys { crypto.SecureWipe(w.keys[i].SecretKeys[j][:]) } } for i := range w.seeds { crypto.SecureWipe(w.seeds[i][:]) } crypto.SecureWipe(w.primarySeed[:]) w.seeds = w.seeds[:0] } // Encrypted returns whether or not the wallet has been encrypted. func (w *Wallet) Encrypted() bool { w.mu.Lock() defer w.mu.Unlock() if build.DEBUG && w.unlocked && len(w.persist.EncryptionVerification) == 0 { panic("wallet is both unlocked and unencrypted") } return len(w.persist.EncryptionVerification) != 0 } // Encrypt will encrypt the wallet using the input key. Upon encryption, a // primary seed will be created for the wallet (no seed exists prior to this // point). If the key is blank, then the hash of the seed that is generated // will be used as the key. The wallet will still be locked after encryption. // // Encrypt can only be called once throughout the life of the wallet, and will // return an error on subsequent calls (even after restarting the wallet). To // reset the wallet, the wallet files must be moved to a different directory or // deleted. func (w *Wallet) Encrypt(masterKey crypto.TwofishKey) (modules.Seed, error) { if err := w.tg.Add(); err != nil { return modules.Seed{}, err } defer w.tg.Done() w.mu.Lock() defer w.mu.Unlock() return w.initEncryption(masterKey) } // Unlocked indicates whether the wallet is locked or unlocked. func (w *Wallet) Unlocked() bool { w.mu.RLock() defer w.mu.RUnlock() return w.unlocked } // Lock will erase all keys from memory and prevent the wallet from spending // coins until it is unlocked. func (w *Wallet) Lock() error { w.mu.Lock() defer w.mu.Unlock() if !w.unlocked { return modules.ErrLockedWallet } w.log.Println("INFO: Locking wallet.") // Wipe all of the seeds and secret keys, they will be replaced upon // calling 'Unlock' again. w.wipeSecrets() w.unlocked = false return nil } // Unlock will decrypt the wallet seed and load all of the addresses into // memory. func (w *Wallet) Unlock(masterKey crypto.TwofishKey) error { // By having the wallet's ThreadGroup track the Unlock method, we ensure // that Unlock will never unlock the wallet once the ThreadGroup has been // stopped. Without this precaution, the wallet's Close method would be // unsafe because it would theoretically be possible for another function // to Unlock the wallet in the short interval after Close calls w.Lock // and before Close calls w.mu.Lock. if err := w.tg.Add(); err != nil { return err } defer w.tg.Done() w.log.Println("INFO: Unlocking wallet.") // Initialize all of the keys in the wallet under a lock. While holding the // lock, also grab the subscriber status. return w.managedUnlock(masterKey) } Sia-1.0.4/modules/wallet/encrypt_test.go000066400000000000000000000152471302033070500202440ustar00rootroot00000000000000package wallet import ( "bytes" "crypto/rand" "path/filepath" "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // postEncryptionTesting runs a series of checks on the wallet after it has // been encrypted, to make sure that locking, unlocking, and spending after // unlocking are all happening in the correct order and returning the correct // errors. func postEncryptionTesting(m modules.TestMiner, w *Wallet, masterKey crypto.TwofishKey) { if !w.Encrypted() { panic("wallet is not encrypted when starting postEncryptionTesting") } if w.Unlocked() { panic("wallet is unlocked when starting postEncryptionTesting") } if len(w.seeds) != 0 { panic("wallet has seeds in it when startin postEncryptionTesting") } // Try unlocking and using the wallet. err := w.Unlock(masterKey) if err != nil { panic(err) } err = w.Unlock(masterKey) if err != errAlreadyUnlocked { panic(err) } // Mine enough coins so that a balance appears (and some buffer for the // send later). for i := types.BlockHeight(0); i <= types.MaturityDelay+1; i++ { _, err := m.AddBlock() if err != nil { panic(err) } } siacoinBal, _, _ := w.ConfirmedBalance() if siacoinBal.Cmp64(0) <= 0 { panic("wallet balance reported as 0 after maturing some mined blocks") } err = w.Unlock(masterKey) if err != errAlreadyUnlocked { panic(err) } // Lock, unlock, and trying using the wallet some more. err = w.Lock() if err != nil { panic(err) } err = w.Lock() if err != modules.ErrLockedWallet { panic(err) } err = w.Unlock(crypto.TwofishKey{}) if err != modules.ErrBadEncryptionKey { panic(err) } err = w.Unlock(masterKey) if err != nil { panic(err) } // Verify that the secret keys have been restored by sending coins to the // void. Send more coins than are received by mining a block. _, err = w.SendSiacoins(types.CalculateCoinbase(0), types.UnlockHash{}) if err != nil { panic(err) } _, err = m.AddBlock() if err != nil { panic(err) } siacoinBal2, _, _ := w.ConfirmedBalance() if siacoinBal2.Cmp(siacoinBal) >= 0 { panic("balance did not increase") } } // TestIntegrationPreEncryption checks that the wallet operates as expected // prior to encryption. func TestIntegrationPreEncryption(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createBlankWalletTester("TestIntegrationPreEncryption") if err != nil { t.Fatal(err) } defer wt.closeWt() // Check that the wallet knows it's not encrypted. if wt.wallet.Encrypted() { t.Error("wallet is reporting that it has been encrypted") } err = wt.wallet.Lock() if err != modules.ErrLockedWallet { t.Fatal(err) } err = wt.wallet.Unlock(crypto.TwofishKey{}) if err != errUnencryptedWallet { t.Fatal(err) } // Create a second wallet using the same directory - make sure that if any // files have been created, the wallet is still being treated as new. w1, err := New(wt.cs, wt.tpool, filepath.Join(wt.persistDir, modules.WalletDir)) if err != nil { t.Fatal(err) } if w1.Encrypted() { t.Error("wallet is reporting that it has been encrypted when no such action has occurred") } if w1.Unlocked() { t.Error("new wallet is not being treated as locked") } } // TestIntegrationUserSuppliedEncryption probes the encryption process when the // user manually supplies an encryption key. func TestIntegrationUserSuppliedEncryption(t *testing.T) { if testing.Short() { t.SkipNow() } // Create and wallet and user-specified key, then encrypt the wallet and // run post-encryption tests on it. wt, err := createBlankWalletTester("TestIntegrationUserSuppliedEncryption") if err != nil { t.Fatal(err) } defer wt.closeWt() var masterKey crypto.TwofishKey _, err = rand.Read(masterKey[:]) if err != nil { t.Fatal(err) } _, err = wt.wallet.Encrypt(masterKey) if err != nil { t.Error(err) } postEncryptionTesting(wt.miner, wt.wallet, masterKey) } // TestIntegrationBlankEncryption probes the encryption process when the user // supplies a blank encryption key during the encryption process. func TestIntegrationBlankEncryption(t *testing.T) { if testing.Short() { t.SkipNow() } // Create the wallet. wt, err := createBlankWalletTester("TestIntegrationBlankEncryption") if err != nil { t.Fatal(err) } defer wt.closeWt() // Encrypt the wallet using a blank key. seed, err := wt.wallet.Encrypt(crypto.TwofishKey{}) if err != nil { t.Error(err) } // Try unlocking the wallet using a blank key. err = wt.wallet.Unlock(crypto.TwofishKey{}) if err != modules.ErrBadEncryptionKey { t.Fatal(err) } // Try unlocking the wallet using the correct key. err = wt.wallet.Unlock(crypto.TwofishKey(crypto.HashObject(seed))) if err != nil { t.Fatal(err) } err = wt.wallet.Lock() if err != nil { t.Fatal(err) } postEncryptionTesting(wt.miner, wt.wallet, crypto.TwofishKey(crypto.HashObject(seed))) } // TestLock checks that lock correctly wipes keys when locking the wallet, // while still being able to track the balance of the wallet. func TestLock(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestLock") if err != nil { t.Fatal(err) } defer wt.closeWt() // Grab a block for work - miner will not supply blocks after the wallet // has been locked, and the test needs to mine a block after locking the // wallet to verify that the balance reporting of a locked wallet is // correct. block, target, err := wt.miner.BlockForWork() if err != nil { t.Fatal(err) } // Lock the wallet. siacoinBalance, _, _ := wt.wallet.ConfirmedBalance() err = wt.wallet.Lock() if err != nil { t.Error(err) } // Compare to the original balance. siacoinBalance2, _, _ := wt.wallet.ConfirmedBalance() if !siacoinBalance2.Equals(siacoinBalance) { t.Error("siacoin balance reporting changed upon closing the wallet") } // Check that the keys and seeds were wiped. wipedKey := make([]byte, crypto.SecretKeySize) for _, key := range wt.wallet.keys { for i := range key.SecretKeys { if !bytes.Equal(wipedKey, key.SecretKeys[i][:]) { t.Error("Key was not wiped after closing the wallet") } } } if len(wt.wallet.seeds) != 0 { t.Error("seeds not wiped from wallet") } if !bytes.Equal(wipedKey[:crypto.EntropySize], wt.wallet.primarySeed[:]) { t.Error("primary seed not wiped from memory") } // Solve the block generated earlier and add it to the consensus set, this // should boost the balance of the wallet. solvedBlock, _ := wt.miner.SolveBlock(block, target) err = wt.cs.AcceptBlock(solvedBlock) if err != nil { t.Fatal(err) } siacoinBalance3, _, _ := wt.wallet.ConfirmedBalance() if siacoinBalance3.Cmp(siacoinBalance2) <= 0 { t.Error("balance should increase after a block was mined") } } Sia-1.0.4/modules/wallet/money.go000066400000000000000000000100311302033070500166320ustar00rootroot00000000000000package wallet import ( "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/types" ) // sortedOutputs is a struct containing a slice of siacoin outputs and their // corresponding ids. sortedOutputs can be sorted using the sort package. type sortedOutputs struct { ids []types.SiacoinOutputID outputs []types.SiacoinOutput } // ConfirmedBalance returns the balance of the wallet according to all of the // confirmed transactions. func (w *Wallet) ConfirmedBalance() (siacoinBalance types.Currency, siafundBalance types.Currency, siafundClaimBalance types.Currency) { w.mu.Lock() defer w.mu.Unlock() for _, sco := range w.siacoinOutputs { siacoinBalance = siacoinBalance.Add(sco.Value) } for _, sfo := range w.siafundOutputs { siafundBalance = siafundBalance.Add(sfo.Value) siafundClaimBalance = siafundClaimBalance.Add(w.siafundPool.Sub(sfo.ClaimStart).Mul(sfo.Value).Div(types.SiafundCount)) } return } // UnconfirmedBalance returns the number of outgoing and incoming siacoins in // the unconfirmed transaction set. Refund outputs are included in this // reporting. func (w *Wallet) UnconfirmedBalance() (outgoingSiacoins types.Currency, incomingSiacoins types.Currency) { w.mu.Lock() defer w.mu.Unlock() for _, upt := range w.unconfirmedProcessedTransactions { for _, input := range upt.Inputs { if input.FundType == types.SpecifierSiacoinInput && input.WalletAddress { outgoingSiacoins = outgoingSiacoins.Add(input.Value) } } for _, output := range upt.Outputs { if output.FundType == types.SpecifierSiacoinOutput && output.WalletAddress { incomingSiacoins = incomingSiacoins.Add(output.Value) } } } return } // SendSiacoins creates a transaction sending 'amount' to 'dest'. The transaction // is submitted to the transaction pool and is also returned. func (w *Wallet) SendSiacoins(amount types.Currency, dest types.UnlockHash) ([]types.Transaction, error) { if err := w.tg.Add(); err != nil { return nil, err } defer w.tg.Done() tpoolFee := types.SiacoinPrecision.Mul64(10) // TODO: better fee algo. output := types.SiacoinOutput{ Value: amount, UnlockHash: dest, } txnBuilder := w.StartTransaction() err := txnBuilder.FundSiacoins(amount.Add(tpoolFee)) if err != nil { return nil, err } txnBuilder.AddMinerFee(tpoolFee) txnBuilder.AddSiacoinOutput(output) txnSet, err := txnBuilder.Sign(true) if err != nil { return nil, err } err = w.tpool.AcceptTransactionSet(txnSet) if err != nil { return nil, err } return txnSet, nil } // SendSiafunds creates a transaction sending 'amount' to 'dest'. The transaction // is submitted to the transaction pool and is also returned. func (w *Wallet) SendSiafunds(amount types.Currency, dest types.UnlockHash) ([]types.Transaction, error) { if err := w.tg.Add(); err != nil { return nil, err } defer w.tg.Done() tpoolFee := types.SiacoinPrecision.Mul64(10) // TODO: better fee algo. output := types.SiafundOutput{ Value: amount, UnlockHash: dest, } txnBuilder := w.StartTransaction() err := txnBuilder.FundSiacoins(tpoolFee) if err != nil { return nil, err } err = txnBuilder.FundSiafunds(amount) if err != nil { return nil, err } txnBuilder.AddMinerFee(tpoolFee) txnBuilder.AddSiafundOutput(output) txnSet, err := txnBuilder.Sign(true) if err != nil { return nil, err } err = w.tpool.AcceptTransactionSet(txnSet) if err != nil { return nil, err } return txnSet, nil } // Len returns the number of elements in the sortedOutputs struct. func (so sortedOutputs) Len() int { if build.DEBUG && len(so.ids) != len(so.outputs) { panic("sortedOutputs object is corrupt") } return len(so.ids) } // Less returns whether element 'i' is less than element 'j'. The currency // value of each output is used for comparison. func (so sortedOutputs) Less(i, j int) bool { return so.outputs[i].Value.Cmp(so.outputs[j].Value) < 0 } // Swap swaps two elements in the sortedOutputs set. func (so sortedOutputs) Swap(i, j int) { so.ids[i], so.ids[j] = so.ids[j], so.ids[i] so.outputs[i], so.outputs[j] = so.outputs[j], so.outputs[i] } Sia-1.0.4/modules/wallet/money_test.go000066400000000000000000000134171302033070500177040ustar00rootroot00000000000000package wallet import ( "sort" "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // TestSendSiacoins probes the SendSiacoins method of the wallet. func TestSendSiacoins(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestSendSiacoins") if err != nil { t.Fatal(err) } defer wt.closeWt() // Get the initial balance - should be 1 block. The unconfirmed balances // should be 0. confirmedBal, _, _ := wt.wallet.ConfirmedBalance() unconfirmedOut, unconfirmedIn := wt.wallet.UnconfirmedBalance() if !confirmedBal.Equals(types.CalculateCoinbase(1)) { t.Error("unexpected confirmed balance") } if !unconfirmedOut.Equals(types.ZeroCurrency) { t.Error("unconfirmed balance should be 0") } if !unconfirmedIn.Equals(types.ZeroCurrency) { t.Error("unconfirmed balance should be 0") } // Send 5000 hastings. The wallet will automatically add a fee. Outgoing // unconfirmed siacoins - incoming unconfirmed siacoins should equal 5000 + // fee. tpoolFee := types.SiacoinPrecision.Mul64(10) _, err = wt.wallet.SendSiacoins(types.NewCurrency64(5000), types.UnlockHash{}) if err != nil { t.Fatal(err) } confirmedBal2, _, _ := wt.wallet.ConfirmedBalance() unconfirmedOut2, unconfirmedIn2 := wt.wallet.UnconfirmedBalance() if !confirmedBal2.Equals(confirmedBal) { t.Error("confirmed balance changed without introduction of blocks") } if !unconfirmedOut2.Equals(unconfirmedIn2.Add(types.NewCurrency64(5000)).Add(tpoolFee)) { t.Error("sending siacoins appears to be ineffective") } // Move the balance into the confirmed set. b, _ := wt.miner.FindBlock() err = wt.cs.AcceptBlock(b) if err != nil { t.Fatal(err) } confirmedBal3, _, _ := wt.wallet.ConfirmedBalance() unconfirmedOut3, unconfirmedIn3 := wt.wallet.UnconfirmedBalance() if !confirmedBal3.Equals(confirmedBal2.Add(types.CalculateCoinbase(2)).Sub(types.NewCurrency64(5000)).Sub(tpoolFee)) { t.Error("confirmed balance did not adjust to the expected value") } if !unconfirmedOut3.Equals(types.ZeroCurrency) { t.Error("unconfirmed balance should be 0") } if !unconfirmedIn3.Equals(types.ZeroCurrency) { t.Error("unconfirmed balance should be 0") } } // TestIntegrationSendOverUnder sends too many siacoins, resulting in an error, // followed by sending few enough siacoins that the send should complete. // // This test is here because of a bug found in production where the wallet // would mark outputs as spent before it knew that there was enough money to // complete the transaction. This meant that, after trying to send too many // coins, all outputs got marked 'sent'. This test reproduces those conditions // to ensure it does not happen again. func TestIntegrationSendOverUnder(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestIntegrationSpendOverUnder") if err != nil { t.Fatal(err) } defer wt.closeWt() // Spend too many siacoins. tooManyCoins := types.SiacoinPrecision.Mul64(1e12) _, err = wt.wallet.SendSiacoins(tooManyCoins, types.UnlockHash{}) if err != modules.ErrLowBalance { t.Error("low balance err not returned after attempting to send too many coins") } // Spend a reasonable amount of siacoins. reasonableCoins := types.SiacoinPrecision.Mul64(100e3) _, err = wt.wallet.SendSiacoins(reasonableCoins, types.UnlockHash{}) if err != nil { t.Error("unexpected error: ", err) } } // TestIntegrationSpendHalfHalf spends more than half of the coins, and then // more than half of the coins again, to make sure that the wallet is not // reusing outputs that it has already spent. func TestIntegrationSpendHalfHalf(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestIntegrationSpendHalfHalf") if err != nil { t.Fatal(err) } defer wt.closeWt() // Spend more than half of the coins twice. halfPlus := types.SiacoinPrecision.Mul64(200e3) _, err = wt.wallet.SendSiacoins(halfPlus, types.UnlockHash{}) if err != nil { t.Error("unexpected error: ", err) } _, err = wt.wallet.SendSiacoins(halfPlus, types.UnlockHash{1}) if err != modules.ErrIncompleteTransactions { t.Error("wallet appears to be reusing outputs when building transactions: ", err) } } // TestIntegrationSpendUnconfirmed spends an unconfirmed siacoin output. func TestIntegrationSpendUnconfirmed(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestIntegrationSpendUnconfirmed") if err != nil { t.Fatal(err) } defer wt.closeWt() // Spend the only output. halfPlus := types.SiacoinPrecision.Mul64(200e3) _, err = wt.wallet.SendSiacoins(halfPlus, types.UnlockHash{}) if err != nil { t.Error("unexpected error: ", err) } someMore := types.SiacoinPrecision.Mul64(75e3) _, err = wt.wallet.SendSiacoins(someMore, types.UnlockHash{1}) if err != nil { t.Error("wallet appears to be struggling to spend unconfirmed outputs") } } // TestIntegrationSortedOutputsSorting checks that the outputs are being correctly sorted // by the currency value. func TestIntegrationSortedOutputsSorting(t *testing.T) { if testing.Short() { t.SkipNow() } so := sortedOutputs{ ids: []types.SiacoinOutputID{{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, outputs: []types.SiacoinOutput{ {Value: types.NewCurrency64(2)}, {Value: types.NewCurrency64(3)}, {Value: types.NewCurrency64(4)}, {Value: types.NewCurrency64(7)}, {Value: types.NewCurrency64(6)}, {Value: types.NewCurrency64(0)}, {Value: types.NewCurrency64(1)}, {Value: types.NewCurrency64(5)}, }, } sort.Sort(so) expectedIDSorting := []types.SiacoinOutputID{{5}, {6}, {0}, {1}, {2}, {7}, {4}, {3}} for i := uint64(0); i < 8; i++ { if so.ids[i] != expectedIDSorting[i] { t.Error("an id is out of place: ", i) } if !so.outputs[i].Value.Equals64(i) { t.Error("a value is out of place: ", i) } } } Sia-1.0.4/modules/wallet/persist.go000066400000000000000000000120401302033070500171760ustar00rootroot00000000000000package wallet import ( "crypto/rand" "os" "path/filepath" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" ) const ( logFile = modules.WalletDir + ".log" settingsFileSuffix = ".json" settingsFile = modules.WalletDir + settingsFileSuffix encryptionVerificationLen = 32 ) var ( settingsMetadata = persist.Metadata{ Header: "Wallet Settings", Version: "0.4.0", } seedMetadata = persist.Metadata{ Header: "Wallet Seed", Version: "0.4.0", } ) // SpendableKeyFile stores an encrypted spendable key on disk. type SpendableKeyFile struct { UID UniqueID EncryptionVerification crypto.Ciphertext SpendableKey crypto.Ciphertext } // WalletPersist contains all data that persists on disk during wallet // operation. type WalletPersist struct { // EncryptionVerification is an encrypted string that, when decrypted, is // 32 '0' bytes. The UID is used to prevent leaking information in the // event that the same key gets used for multiple wallets. UID UniqueID EncryptionVerification crypto.Ciphertext // The primary seed is used to generate new addresses as they are required. // All addresses are tracked and spendable. Only modules.PublicKeysPerSeed // keys/addresses can be created per seed, after which a new seed will need // to be generated. PrimarySeedFile SeedFile PrimarySeedProgress uint64 // AuxiliarySeedFiles is a set of seeds that the wallet can spend from, but is // no longer using to generate addresses. The primary use case is loading // backups in the event of lost files or coins. All auxiliary seeds are // encrypted using the primary seed encryption password. AuxiliarySeedFiles []SeedFile // UnseededKeys are list of spendable keys that were not generated by a // random seed. UnseededKeys []SpendableKeyFile } // loadSettings reads the wallet's settings from the wallet's settings file, // overwriting the settings object in memory. loadSettings should only be // called at startup. func (w *Wallet) loadSettings() error { return persist.LoadFile(settingsMetadata, &w.persist, filepath.Join(w.persistDir, settingsFile)) } // saveSettings writes the wallet's settings to the wallet's settings file, // replacing the existing file. func (w *Wallet) saveSettings() error { return persist.SaveFile(settingsMetadata, w.persist, filepath.Join(w.persistDir, settingsFile)) } // saveSettingsSync writes the wallet's settings to the wallet's settings file, // replacing the existing file, and then syncs to disk. func (w *Wallet) saveSettingsSync() error { return persist.SaveFileSync(settingsMetadata, w.persist, filepath.Join(w.persistDir, settingsFile)) } // initSettings creates the settings object at startup. If a settings file // exists, the settings file will be loaded into memory. If the settings file // does not exist, a new.persist file will be created. func (w *Wallet) initSettings() error { // Check if the settings file exists, if not create it. settingsFilename := filepath.Join(w.persistDir, settingsFile) _, err := os.Stat(settingsFilename) if os.IsNotExist(err) { _, err = rand.Read(w.persist.UID[:]) if err != nil { return err } return w.saveSettings() } else if err != nil { return err } // Load the settings file if it does exist. return w.loadSettings() } // initPersist loads all of the wallet's persistence files into memory, // creating them if they do not exist. func (w *Wallet) initPersist() error { // Create a directory for the wallet without overwriting an existing // directory. err := os.MkdirAll(w.persistDir, 0700) if err != nil { return err } // Start logging. w.log, err = persist.NewFileLogger(filepath.Join(w.persistDir, logFile)) if err != nil { return err } // Load the settings file. err = w.initSettings() if err != nil { return err } return nil } // createBackup creates a backup file at the desired filepath. func (w *Wallet) createBackup(backupFilepath string) error { return persist.SaveFileSync(settingsMetadata, w.persist, backupFilepath) } // CreateBackup creates a backup file at the desired filepath. func (w *Wallet) CreateBackup(backupFilepath string) error { if err := w.tg.Add(); err != nil { return err } defer w.tg.Done() w.mu.Lock() defer w.mu.Unlock() return w.createBackup(backupFilepath) } /* // LoadBackup loads a backup file from the provided filepath. The backup file // primary seed is loaded as an auxiliary seed. func (w *Wallet) LoadBackup(masterKey, backupMasterKey crypto.TwofishKey, backupFilepath string) error { if err := w.tg.Add(); err != nil { return err } defer w.tg.Done() lockID := w.mu.Lock() defer w.mu.Unlock(lockID) // Load all of the seed files, check for duplicates, re-encrypt them (but // keep the UID), and add them to the WalletPersist object) var backupPersist WalletPersist err := persist.LoadFile(settingsMetadata, &backupPersist, backupFilepath) if err != nil { return err } backupSeeds := append(backupPersist.AuxiliarySeedFiles, backupPersist.PrimarySeedFile) TODO: more } */ Sia-1.0.4/modules/wallet/persist_legacy.go000066400000000000000000000120111302033070500205200ustar00rootroot00000000000000package wallet /* import ( "path/filepath" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/types" ) // a savedKey contains a single-signature key and all the tools needed to spend // outputs at its address. type savedKey struct { SecretKey crypto.SecretKey UnlockConditions types.UnlockConditions Visible bool } // saveKeys saves the current set of keys known to the wallet to a file. func (w *Wallet) saveKeys(filepath string) error { // Convert the key map to a slice and write to disk. keySlice := make([]savedKey, 0, len(w.keys)) for _, key := range w.keys { _, exists := w.visibleAddresses[key.unlockConditions.UnlockHash()] keySlice = append(keySlice, savedKey{key.secretKey, key.unlockConditions, exists}) } return encoding.WriteFile(filepath, keySlice) } // saveSiafundTracking save the addresses that track siafunds. func (w *Wallet) saveSiafundTracking(filepath string) error { // Put the siafund tracking addresses into a slice and write to disk. siafundSlice := make([]types.UnlockHash, 0, len(w.siafundAddresses)) for sa, _ := range w.siafundAddresses { siafundSlice = append(siafundSlice, sa) } return encoding.WriteFile(filepath, siafundSlice) } // save writes the contents of a wallet to a file. func (w *Wallet) save() error { // Save the siacoin keys to disk. err := w.saveKeys(filepath.Join(w.persistDir, "wallet.backup")) if err != nil { return err } err = w.saveKeys(filepath.Join(w.persistDir, "wallet.dat")) if err != nil { return err } // outputs.dat is intentionally a bit of a misleading name. If I called it // 'siafunds.dat' or something similar, people might think it's okay to // delete their siafund keys, which is NOT okay. Instead of potentially // having this confusion, I chose a less suggestive name. err = w.saveSiafundTracking(filepath.Join(w.persistDir, "outputs.backup")) if err != nil { return err } // Save the primary copy. err = w.saveSiafundTracking(filepath.Join(w.persistDir, "outputs.dat")) if err != nil { return err } return nil } // loadKeys takes a set of keys and loads them into the wallet. func (w *Wallet) loadKeys(savedKeys []savedKey) error { height := w.consensusHeight for _, skey := range savedKeys { // Skip this key if it's already known to the wallet. _, exists := w.keys[skey.UnlockConditions.UnlockHash()] if exists { continue } // Create an entry in w.keys for each savedKey. w.keys[skey.UnlockConditions.UnlockHash()] = &key{ spendable: height >= skey.UnlockConditions.Timelock, unlockConditions: skey.UnlockConditions, secretKey: skey.SecretKey, outputs: make(map[types.SiacoinOutputID]*knownOutput), } // If Timelock != 0, also add to set of timelockedKeys. if tl := skey.UnlockConditions.Timelock; tl != 0 { w.timelockedKeys[tl] = append(w.timelockedKeys[tl], skey.UnlockConditions.UnlockHash()) } if skey.Visible { w.visibleAddresses[skey.UnlockConditions.UnlockHash()] = struct{}{} } } // If there are no visible addresses, create one. if len(w.visibleAddresses) == 0 { _, _, err := w.coinAddress(true) if err != nil { return err } } return nil } // loadWallet pulls a wallet from disk into memory, merging it with whatever // wallet is already in memory. The result is a combined wallet that has all of // the addresses. func (w *Wallet) loadWallet(filepath string) error { var savedKeys []savedKey err := encoding.ReadFile(filepath, &savedKeys) if err != nil { return err } err = w.loadKeys(savedKeys) if err != nil { return err } return nil } // loadSiafundTracking loads siafund addresses for tracking. func (w *Wallet) loadSiafundTracking(filepath string) error { // Load the siafunds file, which is intentionally called 'outputs.dat'. var siafundAddresses []types.UnlockHash err := encoding.ReadFile(filepath, &siafundAddresses) if err != nil { return err } // Load the addresses into the wallet. for _, sa := range siafundAddresses { w.siafundAddresses[sa] = struct{}{} } return nil } // load reads the contents of a wallet from a file. func (w *Wallet) load() error { err := w.loadWallet(filepath.Join(w.persistDir, "wallet.dat")) if err != nil { // try loading the backup // TODO: display/log a warning err = w.loadWallet(filepath.Join(w.persistDir, "wallet.backup")) if err != nil { return err } } // Load the siafunds file, which is intentionally called 'outputs.dat'. err = w.loadSiafundTracking(filepath.Join(w.persistDir, "outputs.dat")) if err != nil { // try loading the backup // TODO: display/log a warning? err = w.loadSiafundTracking(filepath.Join(w.persistDir, "outputs.backup")) if err != nil { return err } } return w.save() } // MergeWallet merges another wallet with the already-loaded wallet, creating a // new wallet that contains all of the addresses from each. This is useful for // loading backups. func (w *Wallet) MergeWallet(filepath string) error { err := w.loadWallet(filepath) if err != nil { return err } return w.save() } */ Sia-1.0.4/modules/wallet/seed.go000066400000000000000000000213261302033070500164340ustar00rootroot00000000000000package wallet import ( "bytes" "crypto/rand" "errors" "path/filepath" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/types" ) const ( seedFilePrefix = "Sia Wallet Encrypted Backup Seed - " seedFileSuffix = ".seed" ) var ( errAddressExhaustion = errors.New("current seed has used all available addresses") errKnownSeed = errors.New("seed is already known") ) type ( // UniqueID is a unique id randomly generated and put at the front of every // persistence object. It is used to make sure that a different encryption // key can be used for every persistence object. UniqueID [crypto.EntropySize]byte // SeedFile stores an encrypted wallet seed on disk. SeedFile struct { UID UniqueID EncryptionVerification crypto.Ciphertext Seed crypto.Ciphertext } ) // generateUnlockConditions provides the unlock conditions that would be // automatically generated from the input public key. func generateUnlockConditions(pk crypto.PublicKey) types.UnlockConditions { return types.UnlockConditions{ PublicKeys: []types.SiaPublicKey{{ Algorithm: types.SignatureEd25519, Key: pk[:], }}, SignaturesRequired: 1, } } // generateSpendableKey creates the keys and unlock conditions a given index of a // seed. func generateSpendableKey(seed modules.Seed, index uint64) spendableKey { // Generate the keys and unlock conditions. entropy := crypto.HashAll(seed, index) sk, pk := crypto.GenerateKeyPairDeterministic(entropy) return spendableKey{ UnlockConditions: generateUnlockConditions(pk), SecretKeys: []crypto.SecretKey{sk}, } } // encryptAndSaveSeedFile encrypts and saves a seed file. func (w *Wallet) encryptAndSaveSeedFile(masterKey crypto.TwofishKey, seed modules.Seed) (SeedFile, error) { var sf SeedFile _, err := rand.Read(sf.UID[:]) if err != nil { return SeedFile{}, err } sek := uidEncryptionKey(masterKey, sf.UID) plaintextVerification := make([]byte, encryptionVerificationLen) sf.EncryptionVerification, err = sek.EncryptBytes(plaintextVerification) if err != nil { return SeedFile{}, err } sf.Seed, err = sek.EncryptBytes(seed[:]) if err != nil { return SeedFile{}, err } seedFilename := filepath.Join(w.persistDir, seedFilePrefix+persist.RandomSuffix()+seedFileSuffix) err = persist.SaveFileSync(seedMetadata, sf, seedFilename) if err != nil { return SeedFile{}, err } return sf, nil } // decryptSeedFile decrypts a seed file using the encryption key. func decryptSeedFile(masterKey crypto.TwofishKey, sf SeedFile) (seed modules.Seed, err error) { // Verify that the provided master key is the correct key. decryptionKey := uidEncryptionKey(masterKey, sf.UID) expectedDecryptedVerification := make([]byte, crypto.EntropySize) decryptedVerification, err := decryptionKey.DecryptBytes(sf.EncryptionVerification) if err != nil { return modules.Seed{}, err } if !bytes.Equal(expectedDecryptedVerification, decryptedVerification) { return modules.Seed{}, modules.ErrBadEncryptionKey } // Decrypt and return the seed. plainSeed, err := decryptionKey.DecryptBytes(sf.Seed) if err != nil { return modules.Seed{}, err } copy(seed[:], plainSeed) return seed, nil } // integrateSeed takes an address seed as input and from that generates // 'publicKeysPerSeed' addresses that the wallet is able to spend. // integrateSeed should not be called with the primary seed. func (w *Wallet) integrateSeed(seed modules.Seed) { for i := uint64(0); i < modules.PublicKeysPerSeed; i++ { // Generate the key and check it is new to the wallet. spendableKey := generateSpendableKey(seed, i) w.keys[spendableKey.UnlockConditions.UnlockHash()] = spendableKey } w.seeds = append(w.seeds, seed) } // recoverSeed integrates a recovery seed into the wallet. func (w *Wallet) recoverSeed(masterKey crypto.TwofishKey, seed modules.Seed) error { // Because the recovery seed does not have a UID, duplication must be // prevented by comparing with the list of decrypted seeds. This can only // occur while the wallet is unlocked. if !w.unlocked { return modules.ErrLockedWallet } // Check that the seed is not already known. for _, wSeed := range w.seeds { if seed == wSeed { return errKnownSeed } } if seed == w.primarySeed { return errKnownSeed } seedFile, err := w.encryptAndSaveSeedFile(masterKey, seed) if err != nil { return err } // Add the seed file to the wallet's set of tracked seeds and save the // wallet settings. w.persist.AuxiliarySeedFiles = append(w.persist.AuxiliarySeedFiles, seedFile) err = w.saveSettingsSync() if err != nil { return err } w.integrateSeed(seed) return nil } // createSeed creates a wallet seed and encrypts it using a key derived from // the master key, then addds it to the wallet as the primary seed, while // making a disk backup. func (w *Wallet) createSeed(masterKey crypto.TwofishKey, seed modules.Seed) error { seedFile, err := w.encryptAndSaveSeedFile(masterKey, seed) if err != nil { return err } w.primarySeed = seed w.persist.PrimarySeedFile = seedFile w.persist.PrimarySeedProgress = 0 // The wallet preloads keys to prevent confusion for people using the same // seed/wallet file in multiple places. for i := uint64(0); i < modules.WalletSeedPreloadDepth; i++ { spendableKey := generateSpendableKey(seed, i) w.keys[spendableKey.UnlockConditions.UnlockHash()] = spendableKey } return w.saveSettingsSync() } // initPrimarySeed loads the primary seed into the wallet. func (w *Wallet) initPrimarySeed(masterKey crypto.TwofishKey) error { seed, err := decryptSeedFile(masterKey, w.persist.PrimarySeedFile) if err != nil { return err } // The wallet preloads keys to prevent confusion when using the same wallet // in multiple places. for i := uint64(0); i < w.persist.PrimarySeedProgress+modules.WalletSeedPreloadDepth; i++ { spendableKey := generateSpendableKey(seed, i) w.keys[spendableKey.UnlockConditions.UnlockHash()] = spendableKey } w.primarySeed = seed w.seeds = append(w.seeds, seed) return nil } // initAuxiliarySeeds scans the wallet folder for wallet seeds. func (w *Wallet) initAuxiliarySeeds(masterKey crypto.TwofishKey) error { for _, seedFile := range w.persist.AuxiliarySeedFiles { seed, err := decryptSeedFile(masterKey, seedFile) if build.DEBUG && err != nil { panic(err) } if err != nil { w.log.Println("UNLOCK: failed to load an auxiliary seed:", err) continue } w.integrateSeed(seed) } return nil } // nextPrimarySeedAddress fetches the next address from the primary seed. func (w *Wallet) nextPrimarySeedAddress() (types.UnlockConditions, error) { // Check that the wallet has been unlocked. if !w.unlocked { return types.UnlockConditions{}, modules.ErrLockedWallet } // Integrate the next key into the wallet, and return the unlock // conditions. Because the wallet preloads keys, the progress used is // 'PrimarySeedProgress+modules.WalletSeedPreloadDepth'. spendableKey := generateSpendableKey(w.primarySeed, w.persist.PrimarySeedProgress+modules.WalletSeedPreloadDepth) w.keys[spendableKey.UnlockConditions.UnlockHash()] = spendableKey w.persist.PrimarySeedProgress++ err := w.saveSettingsSync() if err != nil { return types.UnlockConditions{}, err } return spendableKey.UnlockConditions, nil } // AllSeeds returns a list of all seeds known to and used by the wallet. func (w *Wallet) AllSeeds() ([]modules.Seed, error) { w.mu.Lock() defer w.mu.Unlock() if !w.unlocked { return nil, modules.ErrLockedWallet } return w.seeds, nil } // PrimarySeed returns the decrypted primary seed of the wallet. func (w *Wallet) PrimarySeed() (modules.Seed, uint64, error) { w.mu.Lock() defer w.mu.Unlock() if !w.unlocked { return modules.Seed{}, 0, modules.ErrLockedWallet } return w.primarySeed, w.persist.PrimarySeedProgress, nil } // NextAddress returns an unlock hash that is ready to receive siacoins or // siafunds. The address is generated using the primary address seed. func (w *Wallet) NextAddress() (types.UnlockConditions, error) { if err := w.tg.Add(); err != nil { return types.UnlockConditions{}, err } defer w.tg.Done() w.mu.Lock() defer w.mu.Unlock() return w.nextPrimarySeedAddress() } // LoadSeed will track all of the addresses generated by the input seed, // reclaiming any funds that were lost due to a deleted file or lost encryption // key. An error will be returned if the seed has already been integrated with // the wallet. func (w *Wallet) LoadSeed(masterKey crypto.TwofishKey, seed modules.Seed) error { if err := w.tg.Add(); err != nil { return err } defer w.tg.Done() w.mu.Lock() defer w.mu.Unlock() err := w.checkMasterKey(masterKey) if err != nil { return err } return w.recoverSeed(masterKey, seed) } Sia-1.0.4/modules/wallet/seed_test.go000066400000000000000000000105751302033070500174770ustar00rootroot00000000000000package wallet import ( "bytes" "path/filepath" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" ) // TestPrimarySeed checks that the correct seed is returned when calling // PrimarySeed. func TestPrimarySeed(t *testing.T) { if testing.Short() { t.SkipNow() } // Start with a blank wallet tester. wt, err := createBlankWalletTester("TestPrimarySeed") if err != nil { t.Fatal(err) } defer wt.closeWt() // Create a seed and unlock the wallet. seed, err := wt.wallet.Encrypt(crypto.TwofishKey{}) if err != nil { t.Fatal(err) } err = wt.wallet.Unlock(crypto.TwofishKey(crypto.HashObject(seed))) if err != nil { t.Fatal(err) } // Try getting an address, see that the seed advances correctly. primarySeed, progress, err := wt.wallet.PrimarySeed() if err != nil { t.Fatal(err) } if !bytes.Equal(primarySeed[:], seed[:]) { t.Error("PrimarySeed is returning a value inconsitent with the seed returned by Encrypt") } if progress != 0 { t.Error("primary seed is returning the wrong progress") } _, err = wt.wallet.NextAddress() if err != nil { t.Fatal(err) } _, progress, err = wt.wallet.PrimarySeed() if err != nil { t.Fatal(err) } if progress != 1 { t.Error("primary seed is returning the wrong progress") } // Lock then unlock the wallet and check the responses. err = wt.wallet.Lock() if err != nil { t.Fatal(err) } _, _, err = wt.wallet.PrimarySeed() if err != modules.ErrLockedWallet { t.Error("unexpected err:", err) } err = wt.wallet.Unlock(crypto.TwofishKey(crypto.HashObject(seed))) if err != nil { t.Fatal(err) } primarySeed, progress, err = wt.wallet.PrimarySeed() if err != nil { t.Fatal(err) } if !bytes.Equal(primarySeed[:], seed[:]) { t.Error("PrimarySeed is returning a value inconsitent with the seed returned by Encrypt") } if progress != 1 { t.Error("progress reporting an unexpected value") } } // TestLoadSeed checks that a seed can be successfully recovered from a wallet, // and then remain available on subsequent loads of the wallet. func TestLoadSeed(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestLoadSeed") if err != nil { t.Fatal(err) } defer wt.closeWt() seed, _, err := wt.wallet.PrimarySeed() if err != nil { t.Fatal(err) } allSeeds, err := wt.wallet.AllSeeds() if err != nil { t.Fatal(err) } if len(allSeeds) != 1 { t.Error("AllSeeds should be returning the primary seed.") } if allSeeds[0] != seed { t.Error("AllSeeds returned the wrong seed") } dir := filepath.Join(build.TempDir(modules.WalletDir, "TestLoadSeed - 0"), modules.WalletDir) w, err := New(wt.cs, wt.tpool, dir) if err != nil { t.Fatal(err) } newSeed, err := w.Encrypt(crypto.TwofishKey{}) if err != nil { t.Fatal(err) } err = w.Unlock(crypto.TwofishKey(crypto.HashObject(newSeed))) if err != nil { t.Fatal(err) } // Balance of wallet should be 0. siacoinBal, _, _ := w.ConfirmedBalance() if !siacoinBal.Equals64(0) { t.Error("fresh wallet should not have a balance") } err = w.LoadSeed(crypto.TwofishKey(crypto.HashObject(newSeed)), seed) if err != nil { t.Fatal(err) } allSeeds, err = w.AllSeeds() if err != nil { t.Fatal(err) } if len(allSeeds) != 2 { t.Error("AllSeeds should be returning the primary seed with the recovery seed.") } if !bytes.Equal(allSeeds[0][:], newSeed[:]) { t.Error("AllSeeds returned the wrong seed") } if !bytes.Equal(allSeeds[1][:], seed[:]) { t.Error("AllSeeds returned the wrong seed") } // Rather than worry about a rescan, which isn't implemented and has // synchronization difficulties, just load a new wallet from the same // settings file - the same effect is achieved without the difficulties. w2, err := New(wt.cs, wt.tpool, dir) if err != nil { t.Fatal(err) } err = w2.Unlock(crypto.TwofishKey(crypto.HashObject(newSeed))) if err != nil { t.Fatal(err) } siacoinBal2, _, _ := w2.ConfirmedBalance() if siacoinBal2.Cmp64(0) <= 0 { t.Error("wallet failed to load a seed with money in it") } allSeeds, err = w2.AllSeeds() if err != nil { t.Fatal(err) } if len(allSeeds) != 2 { t.Error("AllSeeds should be returning the primary seed with the recovery seed.") } if !bytes.Equal(allSeeds[0][:], newSeed[:]) { t.Error("AllSeeds returned the wrong seed") } if !bytes.Equal(allSeeds[1][:], seed[:]) { t.Error("AllSeeds returned the wrong seed") } } Sia-1.0.4/modules/wallet/transactionbuilder.go000066400000000000000000000531221302033070500214070ustar00rootroot00000000000000package wallet import ( "bytes" "errors" "sort" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( // errBuilderAlreadySigned indicates that the transaction builder has // already added at least one successful signature to the transaction, // meaning that future calls to Sign will result in an invalid transaction. errBuilderAlreadySigned = errors.New("sign has already been called on this transaction builder, multiple calls can cause issues") ) // transactionBuilder allows transactions to be manually constructed, including // the ability to fund transactions with siacoins and siafunds from the wallet. type transactionBuilder struct { // 'signed' indicates that at least one transaction signature has been // added to the wallet, meaning that future calls to 'Sign' will fail. parents []types.Transaction signed bool transaction types.Transaction newParents []int siacoinInputs []int siafundInputs []int transactionSignatures []int wallet *Wallet } // addSignatures will sign a transaction using a spendable key, with support // for multisig spendable keys. Because of the restricted input, the function // is compatible with both siacoin inputs and siafund inputs. func addSignatures(txn *types.Transaction, cf types.CoveredFields, uc types.UnlockConditions, parentID crypto.Hash, spendKey spendableKey) (newSigIndices []int, err error) { // Try to find the matching secret key for each public key - some public // keys may not have a match. Some secret keys may be used multiple times, // which is why public keys are used as the outer loop. totalSignatures := uint64(0) for i, siaPubKey := range uc.PublicKeys { // Search for the matching secret key to the public key. for j := range spendKey.SecretKeys { pubKey := spendKey.SecretKeys[j].PublicKey() if bytes.Compare(siaPubKey.Key, pubKey[:]) != 0 { continue } // Found the right secret key, add a signature. sig := types.TransactionSignature{ ParentID: parentID, CoveredFields: cf, PublicKeyIndex: uint64(i), } newSigIndices = append(newSigIndices, len(txn.TransactionSignatures)) txn.TransactionSignatures = append(txn.TransactionSignatures, sig) sigIndex := len(txn.TransactionSignatures) - 1 sigHash := txn.SigHash(sigIndex) encodedSig, err := crypto.SignHash(sigHash, spendKey.SecretKeys[j]) if err != nil { return nil, err } txn.TransactionSignatures[sigIndex].Signature = encodedSig[:] // Count that the signature has been added, and break out of the // secret key loop. totalSignatures++ break } // If there are enough signatures to satisfy the unlock conditions, // break out of the outer loop. if totalSignatures == uc.SignaturesRequired { break } } return newSigIndices, nil } // FundSiacoins will add a siacoin input of exactly 'amount' to the // transaction. A parent transaction may be needed to achieve an input with the // correct value. The siacoin input will not be signed until 'Sign' is called // on the transaction builder. func (tb *transactionBuilder) FundSiacoins(amount types.Currency) error { tb.wallet.mu.Lock() defer tb.wallet.mu.Unlock() // Collect a value-sorted set of siacoin outputs. var so sortedOutputs for scoid, sco := range tb.wallet.siacoinOutputs { so.ids = append(so.ids, scoid) so.outputs = append(so.outputs, sco) } // Add all of the unconfirmed outputs as well. for _, upt := range tb.wallet.unconfirmedProcessedTransactions { for i, sco := range upt.Transaction.SiacoinOutputs { // Determine if the output belongs to the wallet. _, exists := tb.wallet.keys[sco.UnlockHash] if !exists { continue } so.ids = append(so.ids, upt.Transaction.SiacoinOutputID(uint64(i))) so.outputs = append(so.outputs, sco) } } sort.Sort(sort.Reverse(so)) // Create and fund a parent transaction that will add the correct amount of // siacoins to the transaction. var fund types.Currency // potentialFund tracks the balance of the wallet including outputs that // have been spent in other unconfirmed transactions recently. This is to // provide the user with a more useful error message in the event that they // are overspending. var potentialFund types.Currency parentTxn := types.Transaction{} var spentScoids []types.SiacoinOutputID for i := range so.ids { scoid := so.ids[i] sco := so.outputs[i] // Check that this output has not recently been spent by the wallet. spendHeight := tb.wallet.spentOutputs[types.OutputID(scoid)] // Prevent an underflow error. allowedHeight := tb.wallet.consensusSetHeight - RespendTimeout if tb.wallet.consensusSetHeight < RespendTimeout { allowedHeight = 0 } if spendHeight > allowedHeight { potentialFund = potentialFund.Add(sco.Value) continue } outputUnlockConditions := tb.wallet.keys[sco.UnlockHash].UnlockConditions if tb.wallet.consensusSetHeight < outputUnlockConditions.Timelock { continue } // Add a siacoin input for this output. sci := types.SiacoinInput{ ParentID: scoid, UnlockConditions: outputUnlockConditions, } parentTxn.SiacoinInputs = append(parentTxn.SiacoinInputs, sci) spentScoids = append(spentScoids, scoid) // Add the output to the total fund fund = fund.Add(sco.Value) potentialFund = potentialFund.Add(sco.Value) if fund.Cmp(amount) >= 0 { break } } if potentialFund.Cmp(amount) >= 0 && fund.Cmp(amount) < 0 { return modules.ErrIncompleteTransactions } if fund.Cmp(amount) < 0 { return modules.ErrLowBalance } // Create and add the output that will be used to fund the standard // transaction. parentUnlockConditions, err := tb.wallet.nextPrimarySeedAddress() if err != nil { return err } exactOutput := types.SiacoinOutput{ Value: amount, UnlockHash: parentUnlockConditions.UnlockHash(), } parentTxn.SiacoinOutputs = append(parentTxn.SiacoinOutputs, exactOutput) // Create a refund output if needed. if !amount.Equals(fund) { refundUnlockConditions, err := tb.wallet.nextPrimarySeedAddress() if err != nil { return err } refundOutput := types.SiacoinOutput{ Value: fund.Sub(amount), UnlockHash: refundUnlockConditions.UnlockHash(), } parentTxn.SiacoinOutputs = append(parentTxn.SiacoinOutputs, refundOutput) } // Sign all of the inputs to the parent trancstion. for _, sci := range parentTxn.SiacoinInputs { _, err := addSignatures(&parentTxn, types.FullCoveredFields, sci.UnlockConditions, crypto.Hash(sci.ParentID), tb.wallet.keys[sci.UnlockConditions.UnlockHash()]) if err != nil { return err } } // Mark the parent output as spent. Must be done after the transaction is // finished because otherwise the txid and output id will change. tb.wallet.spentOutputs[types.OutputID(parentTxn.SiacoinOutputID(0))] = tb.wallet.consensusSetHeight // Add the exact output. newInput := types.SiacoinInput{ ParentID: parentTxn.SiacoinOutputID(0), UnlockConditions: parentUnlockConditions, } tb.newParents = append(tb.newParents, len(tb.parents)) tb.parents = append(tb.parents, parentTxn) tb.siacoinInputs = append(tb.siacoinInputs, len(tb.transaction.SiacoinInputs)) tb.transaction.SiacoinInputs = append(tb.transaction.SiacoinInputs, newInput) // Mark all outputs that were spent as spent. for _, scoid := range spentScoids { tb.wallet.spentOutputs[types.OutputID(scoid)] = tb.wallet.consensusSetHeight } return nil } // FundSiafunds will add a siafund input of exaclty 'amount' to the // transaction. A parent transaction may be needed to achieve an input with the // correct value. The siafund input will not be signed until 'Sign' is called // on the transaction builder. func (tb *transactionBuilder) FundSiafunds(amount types.Currency) error { tb.wallet.mu.Lock() defer tb.wallet.mu.Unlock() // Create and fund a parent transaction that will add the correct amount of // siafunds to the transaction. var fund types.Currency var potentialFund types.Currency parentTxn := types.Transaction{} var spentSfoids []types.SiafundOutputID for sfoid, sfo := range tb.wallet.siafundOutputs { // Check that this output has not recently been spent by the wallet. spendHeight := tb.wallet.spentOutputs[types.OutputID(sfoid)] // Prevent an underflow error. allowedHeight := tb.wallet.consensusSetHeight - RespendTimeout if tb.wallet.consensusSetHeight < RespendTimeout { allowedHeight = 0 } if spendHeight > allowedHeight { potentialFund = potentialFund.Add(sfo.Value) continue } outputUnlockConditions := tb.wallet.keys[sfo.UnlockHash].UnlockConditions if tb.wallet.consensusSetHeight < outputUnlockConditions.Timelock { continue } // Add a siafund input for this output. parentClaimUnlockConditions, err := tb.wallet.nextPrimarySeedAddress() if err != nil { return err } sfi := types.SiafundInput{ ParentID: sfoid, UnlockConditions: outputUnlockConditions, ClaimUnlockHash: parentClaimUnlockConditions.UnlockHash(), } parentTxn.SiafundInputs = append(parentTxn.SiafundInputs, sfi) spentSfoids = append(spentSfoids, sfoid) // Add the output to the total fund fund = fund.Add(sfo.Value) potentialFund = potentialFund.Add(sfo.Value) if fund.Cmp(amount) >= 0 { break } } if potentialFund.Cmp(amount) >= 0 && fund.Cmp(amount) < 0 { return modules.ErrIncompleteTransactions } if fund.Cmp(amount) < 0 { return modules.ErrLowBalance } // Create and add the output that will be used to fund the standard // transaction. parentUnlockConditions, err := tb.wallet.nextPrimarySeedAddress() if err != nil { return err } exactOutput := types.SiafundOutput{ Value: amount, UnlockHash: parentUnlockConditions.UnlockHash(), } parentTxn.SiafundOutputs = append(parentTxn.SiafundOutputs, exactOutput) // Create a refund output if needed. if !amount.Equals(fund) { refundUnlockConditions, err := tb.wallet.nextPrimarySeedAddress() if err != nil { return err } refundOutput := types.SiafundOutput{ Value: fund.Sub(amount), UnlockHash: refundUnlockConditions.UnlockHash(), } parentTxn.SiafundOutputs = append(parentTxn.SiafundOutputs, refundOutput) } // Sign all of the inputs to the parent trancstion. for _, sfi := range parentTxn.SiafundInputs { _, err := addSignatures(&parentTxn, types.FullCoveredFields, sfi.UnlockConditions, crypto.Hash(sfi.ParentID), tb.wallet.keys[sfi.UnlockConditions.UnlockHash()]) if err != nil { return err } } // Add the exact output. claimUnlockConditions, err := tb.wallet.nextPrimarySeedAddress() if err != nil { return err } newInput := types.SiafundInput{ ParentID: parentTxn.SiafundOutputID(0), UnlockConditions: parentUnlockConditions, ClaimUnlockHash: claimUnlockConditions.UnlockHash(), } tb.newParents = append(tb.newParents, len(tb.parents)) tb.parents = append(tb.parents, parentTxn) tb.siafundInputs = append(tb.siafundInputs, len(tb.transaction.SiafundInputs)) tb.transaction.SiafundInputs = append(tb.transaction.SiafundInputs, newInput) // Mark all outputs that were spent as spent. for _, sfoid := range spentSfoids { tb.wallet.spentOutputs[types.OutputID(sfoid)] = tb.wallet.consensusSetHeight } return nil } // AddParents adds a set of parents to the transaction. func (tb *transactionBuilder) AddParents(newParents []types.Transaction) { tb.parents = append(tb.parents, newParents...) } // AddMinerFee adds a miner fee to the transaction, returning the index of the // miner fee within the transaction. func (tb *transactionBuilder) AddMinerFee(fee types.Currency) uint64 { tb.transaction.MinerFees = append(tb.transaction.MinerFees, fee) return uint64(len(tb.transaction.MinerFees) - 1) } // AddSiacoinInput adds a siacoin input to the transaction, returning the index // of the siacoin input within the transaction. When 'Sign' gets called, this // input will be left unsigned. func (tb *transactionBuilder) AddSiacoinInput(input types.SiacoinInput) uint64 { tb.transaction.SiacoinInputs = append(tb.transaction.SiacoinInputs, input) return uint64(len(tb.transaction.SiacoinInputs) - 1) } // AddSiacoinOutput adds a siacoin output to the transaction, returning the // index of the siacoin output within the transaction. func (tb *transactionBuilder) AddSiacoinOutput(output types.SiacoinOutput) uint64 { tb.transaction.SiacoinOutputs = append(tb.transaction.SiacoinOutputs, output) return uint64(len(tb.transaction.SiacoinOutputs) - 1) } // AddFileContract adds a file contract to the transaction, returning the index // of the file contract within the transaction. func (tb *transactionBuilder) AddFileContract(fc types.FileContract) uint64 { tb.transaction.FileContracts = append(tb.transaction.FileContracts, fc) return uint64(len(tb.transaction.FileContracts) - 1) } // AddFileContractRevision adds a file contract revision to the transaction, // returning the index of the file contract revision within the transaction. // When 'Sign' gets called, this revision will be left unsigned. func (tb *transactionBuilder) AddFileContractRevision(fcr types.FileContractRevision) uint64 { tb.transaction.FileContractRevisions = append(tb.transaction.FileContractRevisions, fcr) return uint64(len(tb.transaction.FileContractRevisions) - 1) } // AddStorageProof adds a storage proof to the transaction, returning the index // of the storage proof within the transaction. func (tb *transactionBuilder) AddStorageProof(sp types.StorageProof) uint64 { tb.transaction.StorageProofs = append(tb.transaction.StorageProofs, sp) return uint64(len(tb.transaction.StorageProofs) - 1) } // AddSiafundInput adds a siafund input to the transaction, returning the index // of the siafund input within the transaction. When 'Sign' is called, this // input will be left unsigned. func (tb *transactionBuilder) AddSiafundInput(input types.SiafundInput) uint64 { tb.transaction.SiafundInputs = append(tb.transaction.SiafundInputs, input) return uint64(len(tb.transaction.SiafundInputs) - 1) } // AddSiafundOutput adds a siafund output to the transaction, returning the // index of the siafund output within the transaction. func (tb *transactionBuilder) AddSiafundOutput(output types.SiafundOutput) uint64 { tb.transaction.SiafundOutputs = append(tb.transaction.SiafundOutputs, output) return uint64(len(tb.transaction.SiafundOutputs) - 1) } // AddArbitraryData adds arbitrary data to the transaction, returning the index // of the data within the transaction. func (tb *transactionBuilder) AddArbitraryData(arb []byte) uint64 { tb.transaction.ArbitraryData = append(tb.transaction.ArbitraryData, arb) return uint64(len(tb.transaction.ArbitraryData) - 1) } // AddTransactionSignature adds a transaction signature to the transaction, // returning the index of the signature within the transaction. The signature // should already be valid, and shouldn't sign any of the inputs that were // added by calling 'FundSiacoins' or 'FundSiafunds'. func (tb *transactionBuilder) AddTransactionSignature(sig types.TransactionSignature) uint64 { tb.transaction.TransactionSignatures = append(tb.transaction.TransactionSignatures, sig) return uint64(len(tb.transaction.TransactionSignatures) - 1) } // Drop discards all of the outputs in a transaction, returning them to the // pool so that other transactions may use them. 'Drop' should only be called // if a transaction is both unsigned and will not be used any further. func (tb *transactionBuilder) Drop() { tb.wallet.mu.Lock() defer tb.wallet.mu.Unlock() // Iterate through all parents and the transaction itself and restore all // outputs to the list of available outputs. txns := append(tb.parents, tb.transaction) for _, txn := range txns { for _, sci := range txn.SiacoinInputs { delete(tb.wallet.spentOutputs, types.OutputID(sci.ParentID)) } } tb.parents = nil tb.signed = false tb.transaction = types.Transaction{} tb.newParents = nil tb.siacoinInputs = nil tb.siafundInputs = nil tb.transactionSignatures = nil } // Sign will sign any inputs added by 'FundSiacoins' or 'FundSiafunds' and // return a transaction set that contains all parents prepended to the // transaction. If more fields need to be added, a new transaction builder will // need to be created. // // If the whole transaction flag is set to true, then the whole transaction // flag will be set in the covered fields object. If the whole transaction flag // is set to false, then the covered fields object will cover all fields that // have already been added to the transaction, but will also leave room for // more fields to be added. // // Sign should not be called more than once. If, for some reason, there is an // error while calling Sign, the builder should be dropped. func (tb *transactionBuilder) Sign(wholeTransaction bool) ([]types.Transaction, error) { if tb.signed { return nil, errBuilderAlreadySigned } // Create the coveredfields struct. var coveredFields types.CoveredFields if wholeTransaction { coveredFields = types.CoveredFields{WholeTransaction: true} } else { for i := range tb.transaction.MinerFees { coveredFields.MinerFees = append(coveredFields.MinerFees, uint64(i)) } for i := range tb.transaction.SiacoinInputs { coveredFields.SiacoinInputs = append(coveredFields.SiacoinInputs, uint64(i)) } for i := range tb.transaction.SiacoinOutputs { coveredFields.SiacoinOutputs = append(coveredFields.SiacoinOutputs, uint64(i)) } for i := range tb.transaction.FileContracts { coveredFields.FileContracts = append(coveredFields.FileContracts, uint64(i)) } for i := range tb.transaction.FileContractRevisions { coveredFields.FileContractRevisions = append(coveredFields.FileContractRevisions, uint64(i)) } for i := range tb.transaction.StorageProofs { coveredFields.StorageProofs = append(coveredFields.StorageProofs, uint64(i)) } for i := range tb.transaction.SiafundInputs { coveredFields.SiafundInputs = append(coveredFields.SiafundInputs, uint64(i)) } for i := range tb.transaction.SiafundOutputs { coveredFields.SiafundOutputs = append(coveredFields.SiafundOutputs, uint64(i)) } for i := range tb.transaction.ArbitraryData { coveredFields.ArbitraryData = append(coveredFields.ArbitraryData, uint64(i)) } } // TransactionSignatures don't get covered by the 'WholeTransaction' flag, // and must be covered manually. for i := range tb.transaction.TransactionSignatures { coveredFields.TransactionSignatures = append(coveredFields.TransactionSignatures, uint64(i)) } // For each siacoin input in the transaction that we added, provide a // signature. tb.wallet.mu.Lock() defer tb.wallet.mu.Unlock() for _, inputIndex := range tb.siacoinInputs { input := tb.transaction.SiacoinInputs[inputIndex] key := tb.wallet.keys[input.UnlockConditions.UnlockHash()] newSigIndices, err := addSignatures(&tb.transaction, coveredFields, input.UnlockConditions, crypto.Hash(input.ParentID), key) if err != nil { return nil, err } tb.transactionSignatures = append(tb.transactionSignatures, newSigIndices...) tb.signed = true // Signed is set to true after one successful signature to indicate that future signings can cause issues. } for _, inputIndex := range tb.siafundInputs { input := tb.transaction.SiafundInputs[inputIndex] key := tb.wallet.keys[input.UnlockConditions.UnlockHash()] newSigIndices, err := addSignatures(&tb.transaction, coveredFields, input.UnlockConditions, crypto.Hash(input.ParentID), key) if err != nil { return nil, err } tb.transactionSignatures = append(tb.transactionSignatures, newSigIndices...) tb.signed = true // Signed is set to true after one successful signature to indicate that future signings can cause issues. } // Get the transaction set and delete the transaction from the registry. txnSet := append(tb.parents, tb.transaction) return txnSet, nil } // ViewTransaction returns a transaction-in-progress along with all of its // parents, specified by id. An error is returned if the id is invalid. Note // that ids become invalid for a transaction after 'SignTransaction' has been // called because the transaction gets deleted. func (tb *transactionBuilder) View() (types.Transaction, []types.Transaction) { return tb.transaction, tb.parents } // ViewAdded returns all of the siacoin inputs, siafund inputs, and parent // transactions that have been automatically added by the builder. func (tb *transactionBuilder) ViewAdded() (newParents, siacoinInputs, siafundInputs, transactionSignatures []int) { return tb.newParents, tb.siacoinInputs, tb.siafundInputs, tb.transactionSignatures } // RegisterTransaction takes a transaction and its parents and returns a // TransactionBuilder which can be used to expand the transaction. The most // typical call is 'RegisterTransaction(types.Transaction{}, nil)', which // registers a new transaction without parents. func (w *Wallet) RegisterTransaction(t types.Transaction, parents []types.Transaction) modules.TransactionBuilder { // Create a deep copy of the transaction and parents by encoding them. A // deep copy ensures that there are no pointer or slice related errors - // the builder will be working directly on the transaction, and the // transaction may be in use elsewhere (in this case, the host is using the // transaction. pBytes := encoding.Marshal(parents) var pCopy []types.Transaction err := encoding.Unmarshal(pBytes, &pCopy) if err != nil { panic(err) } tBytes := encoding.Marshal(t) var tCopy types.Transaction err = encoding.Unmarshal(tBytes, &tCopy) if err != nil { panic(err) } return &transactionBuilder{ parents: pCopy, transaction: tCopy, wallet: w, } } // StartTransaction is a convenience function that calls // RegisterTransaction(types.Transaction{}, nil). func (w *Wallet) StartTransaction() modules.TransactionBuilder { return w.RegisterTransaction(types.Transaction{}, nil) } Sia-1.0.4/modules/wallet/transactionbuilder_test.go000066400000000000000000000324711302033070500224520ustar00rootroot00000000000000package wallet import ( "sync" "testing" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // addBlockNoPayout adds a block to the wallet tester that does not have any // payouts. func (wt *walletTester) addBlockNoPayout() error { block, target, err := wt.miner.BlockForWork() if err != nil { return err } // Clear the miner payout so that the wallet is not getting additional // outputs from these blocks. for i := range block.MinerPayouts { block.MinerPayouts[i].UnlockHash = types.UnlockHash{} } // Solve and submit the block. solvedBlock, _ := wt.miner.SolveBlock(block, target) err = wt.cs.AcceptBlock(solvedBlock) if err != nil { return err } return nil } // TestViewAdded checks that 'ViewAdded' returns sane-seeming values when // indicating which elements have been added automatically to a transaction // set. func TestViewAdded(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestViewAdded") if err != nil { t.Fatal(err) } defer wt.closeWt() // Mine an extra block to get more outputs - the wallet is going to be // loading two transactions at the same time. _, err = wt.miner.AddBlock() if err != nil { t.Fatal(err) } // Create a transaction, add money to it, spend the money in a miner fee // but do not sign the transaction. The format of this test mimics the way // that the host-renter protocol behaves when building a file contract // transaction. b := wt.wallet.StartTransaction() txnFund := types.NewCurrency64(100e9) err = b.FundSiacoins(txnFund) if err != nil { t.Fatal(err) } _ = b.AddMinerFee(txnFund) _ = b.AddSiacoinOutput(types.SiacoinOutput{Value: txnFund}) unfinishedTxn, unfinishedParents := b.View() // Create a second builder that extends the first, unsigned transaction. Do // not sign the transaction, but do give the extensions to the original // builder. b2 := wt.wallet.RegisterTransaction(unfinishedTxn, unfinishedParents) err = b2.FundSiacoins(txnFund) if err != nil { t.Fatal(err) } unfinishedTxn2, unfinishedParents2 := b2.View() newParentIndices, newInputIndices, _, _ := b2.ViewAdded() // Add the new elements from b2 to b and sign the transaction, fetching the // signature for b. for _, parentIndex := range newParentIndices { b.AddParents([]types.Transaction{unfinishedParents2[parentIndex]}) } for _, inputIndex := range newInputIndices { b.AddSiacoinInput(unfinishedTxn2.SiacoinInputs[inputIndex]) } // Signing with WholeTransaction=true makes the transaction more brittle to // construction mistakes, meaning that an error is more likely to turn up. set1, err := b.Sign(true) if err != nil { t.Fatal(err) } if set1[len(set1)-1].ID() == unfinishedTxn.ID() { t.Error("seems like there's memory sharing happening between txn calls") } // Set1 should be missing some signatures. err = wt.tpool.AcceptTransactionSet(set1) if err == nil { t.Fatal(err) } unfinishedTxn3, _ := b.View() // Only the new signatures are needed because the previous call to 'View' // included everything else. _, _, _, newTxnSignaturesIndices := b.ViewAdded() // Add the new signatures to b2, and then sign b2's inputs. The resulting // set from b2 should be valid. for _, sigIndex := range newTxnSignaturesIndices { b2.AddTransactionSignature(unfinishedTxn3.TransactionSignatures[sigIndex]) } set2, err := b2.Sign(true) err = wt.tpool.AcceptTransactionSet(set2) if err != nil { t.Fatal(err) } finishedTxn, _ := b2.View() _, _, _, newTxnSignaturesIndices3 := b2.ViewAdded() // Add the new signatures from b2 to the b1 transaction, which should // complete the transaction and create a transaction set in 'b' that is // identical to the transaction set that is in b2. for _, sigIndex := range newTxnSignaturesIndices3 { b.AddTransactionSignature(finishedTxn.TransactionSignatures[sigIndex]) } set3Txn, set3Parents := b.View() err = wt.tpool.AcceptTransactionSet(append(set3Parents, set3Txn)) if err != modules.ErrDuplicateTransactionSet { t.Fatal(err) } } // TestDoubleSignError checks that an error is returned if there is a problem // when trying to call 'Sign' on a transaction twice. func TestDoubleSignError(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestDoubleSignError") if err != nil { t.Fatal(err) } defer wt.closeWt() // Create a transaction, add money to it, and then call sign twice. b := wt.wallet.StartTransaction() txnFund := types.NewCurrency64(100e9) err = b.FundSiacoins(txnFund) if err != nil { t.Fatal(err) } _ = b.AddMinerFee(txnFund) txnSet, err := b.Sign(true) if err != nil { t.Fatal(err) } txnSet2, err := b.Sign(true) if err != errBuilderAlreadySigned { t.Error("the wrong error is being returned after a double call to sign") } if err != nil && txnSet2 != nil { t.Error("errored call to sign did not return a nil txn set") } err = wt.tpool.AcceptTransactionSet(txnSet) if err != nil { t.Fatal(err) } } // TestConcurrentBuilders checks that multiple transaction builders can safely // be opened at the same time, and that they will make valid transactions when // building concurrently. func TestConcurrentBuilders(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestConcurrentBuilders") if err != nil { t.Fatal(err) } defer wt.closeWt() // Mine a few more blocks so that the wallet has lots of outputs to pick // from. for i := 0; i < 5; i++ { _, err := wt.miner.AddBlock() if err != nil { t.Fatal(err) } } // Get a baseline balance for the wallet. startingSCConfirmed, _, _ := wt.wallet.ConfirmedBalance() startingOutgoing, startingIncoming := wt.wallet.UnconfirmedBalance() if !startingOutgoing.IsZero() { t.Fatal(startingOutgoing) } if !startingIncoming.IsZero() { t.Fatal(startingIncoming) } // Create two builders at the same time, then add money to each. builder1 := wt.wallet.StartTransaction() builder2 := wt.wallet.StartTransaction() // Fund each builder with a siacoin output that is smaller than all of the // outputs that the wallet should currently have. funding := types.NewCurrency64(10e3).Mul(types.SiacoinPrecision) err = builder1.FundSiacoins(funding) if err != nil { t.Fatal(err) } err = builder2.FundSiacoins(funding) if err != nil { t.Fatal(err) } // Get a second reading on the wallet's balance. fundedSCConfirmed, _, _ := wt.wallet.ConfirmedBalance() if !startingSCConfirmed.Equals(fundedSCConfirmed) { t.Fatal("confirmed siacoin balance changed when no blocks have been mined", startingSCConfirmed, fundedSCConfirmed) } // Spend the transaction funds on miner fees and the void output. builder1.AddMinerFee(types.NewCurrency64(25).Mul(types.SiacoinPrecision)) builder2.AddMinerFee(types.NewCurrency64(25).Mul(types.SiacoinPrecision)) // Send the money to the void. output := types.SiacoinOutput{Value: types.NewCurrency64(9975).Mul(types.SiacoinPrecision)} builder1.AddSiacoinOutput(output) builder2.AddSiacoinOutput(output) // Sign the transactions and verify that both are valid. tset1, err := builder1.Sign(true) if err != nil { t.Fatal(err) } tset2, err := builder2.Sign(true) if err != nil { t.Fatal(err) } err = wt.tpool.AcceptTransactionSet(tset1) if err != nil { t.Fatal(err) } err = wt.tpool.AcceptTransactionSet(tset2) if err != nil { t.Fatal(err) } // Mine a block to get the transaction sets into the blockchain. _, err = wt.miner.AddBlock() if err != nil { t.Fatal(err) } } // TestConcurrentBuildersSingleOutput probes the behavior when multiple // builders are created at the same time, but there is only a single wallet // output that they end up needing to share. func TestConcurrentBuildersSingleOutput(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestConcurrentBuildersSingleOutput") if err != nil { t.Fatal(err) } defer wt.closeWt() // Mine MaturityDelay blocks on the wallet using blocks that don't give // miner payouts to the wallet, so that all outputs can be condensed into a // single confirmed output. Currently the wallet will be getting a new // output per block because it has mined some blocks that haven't had their // outputs matured. for i := types.BlockHeight(0); i < types.MaturityDelay+1; i++ { err = wt.addBlockNoPayout() if err != nil { t.Fatal(err) } } // Send all coins to a single confirmed output for the wallet. unlockConditions, err := wt.wallet.NextAddress() scBal, _, _ := wt.wallet.ConfirmedBalance() // Use a custom builder so that there is no transaction fee. builder := wt.wallet.StartTransaction() err = builder.FundSiacoins(scBal) if err != nil { t.Fatal(err) } output := types.SiacoinOutput{ Value: scBal, UnlockHash: unlockConditions.UnlockHash(), } builder.AddSiacoinOutput(output) tSet, err := builder.Sign(true) if err != nil { t.Fatal(err) } err = wt.tpool.AcceptTransactionSet(tSet) if err != nil { t.Fatal(err) } // Get the transaction into the blockchain without giving a miner payout to // the wallet. err = wt.addBlockNoPayout() if err != nil { t.Fatal(err) } // Check that there is only one output in the wallet. if len(wt.wallet.siacoinOutputs) != 1 { t.Fatal("wallet is supposed to have only one output", wt.wallet.siacoinOutputs) } // Get a baseline balance for the wallet. startingSCConfirmed, _, _ := wt.wallet.ConfirmedBalance() startingOutgoing, startingIncoming := wt.wallet.UnconfirmedBalance() if !startingOutgoing.IsZero() { t.Fatal(startingOutgoing) } if !startingIncoming.IsZero() { t.Fatal(startingIncoming) } // Create two builders at the same time, then add money to each. builder1 := wt.wallet.StartTransaction() builder2 := wt.wallet.StartTransaction() // Fund each builder with a siacoin output. funding := types.NewCurrency64(10e3).Mul(types.SiacoinPrecision) err = builder1.FundSiacoins(funding) if err != nil { t.Fatal(err) } // This add should fail, blocking the builder from completion. err = builder2.FundSiacoins(funding) if err != modules.ErrIncompleteTransactions { t.Fatal(err) } // Get a second reading on the wallet's balance. fundedSCConfirmed, _, _ := wt.wallet.ConfirmedBalance() if !startingSCConfirmed.Equals(fundedSCConfirmed) { t.Fatal("confirmed siacoin balance changed when no blocks have been mined", startingSCConfirmed, fundedSCConfirmed) } // Spend the transaction funds on miner fees and the void output. builder1.AddMinerFee(types.NewCurrency64(25).Mul(types.SiacoinPrecision)) // Send the money to the void. output = types.SiacoinOutput{Value: types.NewCurrency64(9975).Mul(types.SiacoinPrecision)} builder1.AddSiacoinOutput(output) // Sign the transaction and submit it. tset1, err := builder1.Sign(true) if err != nil { t.Fatal(err) } err = wt.tpool.AcceptTransactionSet(tset1) if err != nil { t.Fatal(err) } // Mine a block to get the transaction sets into the blockchain. _, err = wt.miner.AddBlock() if err != nil { t.Fatal(err) } } // TestParallelBuilders checks that multiple transaction builders can safely be // opened at the same time, and that they will make valid transactions when // building concurrently, using multiple gothreads to manage the builders. func TestParallelBuilders(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestParallelBuilders") if err != nil { t.Fatal(err) } defer wt.closeWt() // Mine a few more blocks so that the wallet has lots of outputs to pick // from. outputsDesired := 10 for i := 0; i < outputsDesired; i++ { _, err := wt.miner.AddBlock() if err != nil { t.Fatal(err) } } // Add MatruityDelay blocks with no payout to make tracking the balance // easier. for i := types.BlockHeight(0); i < types.MaturityDelay+1; i++ { err = wt.addBlockNoPayout() if err != nil { t.Fatal(err) } } // Get a baseline balance for the wallet. startingSCConfirmed, _, _ := wt.wallet.ConfirmedBalance() startingOutgoing, startingIncoming := wt.wallet.UnconfirmedBalance() if !startingOutgoing.IsZero() { t.Fatal(startingOutgoing) } if !startingIncoming.IsZero() { t.Fatal(startingIncoming) } // Create several builders in parallel. var wg sync.WaitGroup funding := types.NewCurrency64(10e3).Mul(types.SiacoinPrecision) for i := 0; i < outputsDesired; i++ { wg.Add(1) go func() { // Create the builder and fund the transaction. builder := wt.wallet.StartTransaction() err := builder.FundSiacoins(funding) if err != nil { t.Fatal(err) } // Spend the transaction funds on miner fees and the void output. builder.AddMinerFee(types.NewCurrency64(25).Mul(types.SiacoinPrecision)) output := types.SiacoinOutput{Value: types.NewCurrency64(9975).Mul(types.SiacoinPrecision)} builder.AddSiacoinOutput(output) // Sign the transactions and verify that both are valid. tset, err := builder.Sign(true) if err != nil { t.Fatal(err) } err = wt.tpool.AcceptTransactionSet(tset) if err != nil { t.Fatal(err) } wg.Done() }() } wg.Wait() // Mine a block to get the transaction sets into the blockchain. err = wt.addBlockNoPayout() if err != nil { t.Fatal(err) } // Check the final balance. endingSCConfirmed, _, _ := wt.wallet.ConfirmedBalance() expected := startingSCConfirmed.Sub(funding.Mul(types.NewCurrency64(uint64(outputsDesired)))) if !expected.Equals(endingSCConfirmed) { t.Fatal("did not get the expected ending balance", expected, endingSCConfirmed, startingSCConfirmed) } } Sia-1.0.4/modules/wallet/transactions.go000066400000000000000000000055021302033070500202220ustar00rootroot00000000000000package wallet import ( "errors" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) var ( errOutOfBounds = errors.New("requesting transactions at unknown confirmation heights") errNoHistoryForAddr = errors.New("no history found for provided address") ) // AddressTransactions returns all of the wallet transactions associated with a // single unlock hash. func (w *Wallet) AddressTransactions(uh types.UnlockHash) (pts []modules.ProcessedTransaction) { w.mu.Lock() defer w.mu.Unlock() for _, pt := range w.processedTransactions { relevant := false for _, input := range pt.Inputs { if input.RelatedAddress == uh { relevant = true break } } for _, output := range pt.Outputs { if output.RelatedAddress == uh { relevant = true break } } if relevant { pts = append(pts, pt) } } return pts } // AddressUnconfirmedHistory returns all of the unconfirmed wallet transactions // related to a specific address. func (w *Wallet) AddressUnconfirmedTransactions(uh types.UnlockHash) (pts []modules.ProcessedTransaction) { w.mu.Lock() defer w.mu.Unlock() // Scan the full list of unconfirmed transactions to see if there are any // related transactions. for _, pt := range w.unconfirmedProcessedTransactions { relevant := false for _, input := range pt.Inputs { if input.RelatedAddress == uh { relevant = true break } } for _, output := range pt.Outputs { if output.RelatedAddress == uh { relevant = true break } } if relevant { pts = append(pts, pt) } } return pts } // Transaction returns the transaction with the given id. 'False' is returned // if the transaction does not exist. func (w *Wallet) Transaction(txid types.TransactionID) (modules.ProcessedTransaction, bool) { w.mu.Lock() defer w.mu.Unlock() pt, exists := w.processedTransactionMap[txid] if !exists { return modules.ProcessedTransaction{}, exists } return *pt, exists } // Transactions returns all transactions relevant to the wallet that were // confirmed in the range [startHeight, endHeight]. func (w *Wallet) Transactions(startHeight, endHeight types.BlockHeight) (pts []modules.ProcessedTransaction, err error) { w.mu.Lock() defer w.mu.Unlock() if startHeight > w.consensusSetHeight || startHeight > endHeight { return nil, errOutOfBounds } if len(w.processedTransactions) == 0 { return nil, nil } for _, pt := range w.processedTransactions { if pt.ConfirmationHeight > endHeight { break } if pt.ConfirmationHeight >= startHeight { pts = append(pts, pt) } } return pts, nil } // UnconfirmedTransactions returns the set of unconfirmed transactions that are // relevant to the wallet. func (w *Wallet) UnconfirmedTransactions() []modules.ProcessedTransaction { w.mu.Lock() defer w.mu.Unlock() return w.unconfirmedProcessedTransactions } Sia-1.0.4/modules/wallet/transactions_test.go000066400000000000000000000072121302033070500212610ustar00rootroot00000000000000package wallet import ( "testing" "github.com/NebulousLabs/Sia/types" ) // TestIntegrationTransactions checks that the transaction history is being // correctly recorded and extended. func TestIntegrationTransactions(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestIntegrationTransactions") if err != nil { t.Fatal(err) } defer wt.closeWt() // Creating the wallet tester results in blocks being mined until the miner // has money, which means types.MaturityDelay+1 blocks are created, and // each block is going to have a transaction (the miner payout) going to // the wallet. txns, err := wt.wallet.Transactions(0, 100) if err != nil { t.Fatal(err) } if len(txns) != int(types.MaturityDelay+1) { t.Error("unexpected transaction history length") } sendTxns, err := wt.wallet.SendSiacoins(types.NewCurrency64(5000), types.UnlockHash{}) if err != nil { t.Fatal(err) } // No more confirmed transactions have been added. txns, err = wt.wallet.Transactions(0, 100) if err != nil { t.Fatal(err) } if len(txns) != int(types.MaturityDelay+1) { t.Error("unexpected transaction history length") } // Two transactions added to unconfirmed pool - 1 to fund the exact output, // and 1 to hold the exact output. if len(wt.wallet.UnconfirmedTransactions()) != 2 { t.Error("was expecting 4 unconfirmed transactions") } b, _ := wt.miner.FindBlock() err = wt.cs.AcceptBlock(b) if err != nil { t.Fatal(err) } // A confirmed transaction was added for the miner payout, and the 2 // transactions that were previously unconfirmed. txns, err = wt.wallet.Transactions(0, 100) if err != nil { t.Fatal(err) } if len(txns) != int(types.MaturityDelay+2+2) { t.Error("unexpected transaction history length") } // Try getting a partial history for just the previous block. txns, err = wt.wallet.Transactions(types.MaturityDelay+3, types.MaturityDelay+3) if err != nil { t.Fatal(err) } // The partial should include one transaction for a block, and 2 for the // send that occurred. if len(txns) != 3 { t.Error(len(txns)) } // Check that transactions can be queried individually. txn, exists := wt.wallet.Transaction(sendTxns[0].ID()) if !exists { t.Fatal("unable to query transaction") } if txn.TransactionID != sendTxns[0].ID() { t.Error("wrong transaction was fetched") } _, exists = wt.wallet.Transaction(types.TransactionID{}) if exists { t.Error("able to query a nonexisting transction") } } // TestIntegrationAddressTransactions checks grabbing the history for a single // address. func TestIntegrationAddressTransactions(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestTransactionHistory") if err != nil { t.Fatal(err) } defer wt.closeWt() // Grab an address and send it money. uc, err := wt.wallet.NextAddress() addr := uc.UnlockHash() if err != nil { t.Fatal(err) } _, err = wt.wallet.SendSiacoins(types.NewCurrency64(5005), addr) if err != nil { t.Fatal(err) } // Check the confirmed balance of the address. addrHist := wt.wallet.AddressTransactions(addr) if len(addrHist) != 0 { t.Error("address should be empty - no confirmed transactions") } if len(wt.wallet.AddressUnconfirmedTransactions(addr)) == 0 { t.Error("addresses unconfirmed transactions should not be empty") } b, _ := wt.miner.FindBlock() err = wt.cs.AcceptBlock(b) if err != nil { t.Fatal(err) } addrHist = wt.wallet.AddressTransactions(addr) if len(addrHist) == 0 { t.Error("address history should have some transactions") } if len(wt.wallet.AddressUnconfirmedTransactions(addr)) != 0 { t.Error("addresses unconfirmed transactions should be empty") } } Sia-1.0.4/modules/wallet/unseeded.go000066400000000000000000000155311302033070500173110ustar00rootroot00000000000000package wallet import ( "bytes" "crypto/rand" "errors" "path/filepath" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" "github.com/NebulousLabs/Sia/types" ) const ( // The header for all siag files. Do not change. Because siag was created // early in development, compatibility with siag requires manually handling // the headers and version instead of using the persist package. SiagFileHeader = "siag" SiagFileExtension = ".siakey" SiagFileVersion = "1.0" ) var ( ErrInconsistentKeys = errors.New("keyfiles provided that are for different addresses") ErrInsufficientKeys = errors.New("not enough keys provided to spend the siafunds") ErrNoKeyfile = errors.New("no keyfile has been presented") ErrUnknownHeader = errors.New("file contains the wrong header") ErrUnknownVersion = errors.New("file has an unknown version number") errAllDuplicates = errors.New("old wallet has no new seeds") errDuplicateSpendableKey = errors.New("key has already been loaded into the wallet") ) // A SiagKeyPair is the struct representation of the bytes that get saved to // disk by siag when a new keyfile is created. type SiagKeyPair struct { Header string Version string Index int // should be uint64 - too late now SecretKey crypto.SecretKey UnlockConditions types.UnlockConditions } // SavedKey033x is the persist structure that was used to save and load private // keys in versions v0.3.3.x for siad. type SavedKey033x struct { SecretKey crypto.SecretKey UnlockConditions types.UnlockConditions Visible bool } // initUnseededKeys loads all of the unseeded keys into the wallet after the // wallet gets unlocked. func (w *Wallet) initUnseededKeys(masterKey crypto.TwofishKey) error { for _, uk := range w.persist.UnseededKeys { // Verify that the decryption key is correct. encKey := uidEncryptionKey(masterKey, uk.UID) expectedDecryptedVerification := make([]byte, crypto.EntropySize) decryptedVerification, err := encKey.DecryptBytes(uk.EncryptionVerification) if err != nil { return err } if !bytes.Equal(expectedDecryptedVerification, decryptedVerification) { return modules.ErrBadEncryptionKey } // Decrypt the spendable key and add it to the wallet. encodedKey, err := encKey.DecryptBytes(uk.SpendableKey) if err != nil { return err } var sk spendableKey err = encoding.Unmarshal(encodedKey, &sk) if err != nil { return err } w.keys[sk.UnlockConditions.UnlockHash()] = sk } return nil } // loadSpendableKey loads a spendable key into the wallet's persist structure. func (w *Wallet) loadSpendableKey(masterKey crypto.TwofishKey, sk spendableKey) error { // Duplication is detected by looking at the set of unlock conditions. If // the wallet is locked, correct deduplication is uncertain. if !w.unlocked { return modules.ErrLockedWallet } // Check for duplicates. _, exists := w.keys[sk.UnlockConditions.UnlockHash()] if exists { return errDuplicateSpendableKey } // TODO: Check that the key is actually spendable. // Create a UID and encryption verification. var skf SpendableKeyFile _, err := rand.Read(skf.UID[:]) if err != nil { return err } encryptionKey := uidEncryptionKey(masterKey, skf.UID) plaintextVerification := make([]byte, encryptionVerificationLen) skf.EncryptionVerification, err = encryptionKey.EncryptBytes(plaintextVerification) if err != nil { return err } // Encrypt and save the key. skf.SpendableKey, err = encryptionKey.EncryptBytes(encoding.Marshal(sk)) if err != nil { return err } w.persist.UnseededKeys = append(w.persist.UnseededKeys, skf) // w.keys[sk.UnlockConditions.UnlockHash()] = sk -> aids with duplicate // detection, but causes db inconsistency. Rescanning is probably the // solution. return nil } // loadSiagKeys loads a set of siag keyfiles into the wallet, so that the // wallet may spend the siafunds. func (w *Wallet) loadSiagKeys(masterKey crypto.TwofishKey, keyfiles []string) error { // Load the keyfiles from disk. if len(keyfiles) < 1 { return ErrNoKeyfile } skps := make([]SiagKeyPair, len(keyfiles)) for i, keyfile := range keyfiles { err := encoding.ReadFile(keyfile, &skps[i]) if err != nil { return err } if skps[i].Header != SiagFileHeader { return ErrUnknownHeader } if skps[i].Version != SiagFileVersion { return ErrUnknownVersion } } // Check that all of the loaded files have the same address, and that there // are enough to create the transaction. baseUnlockHash := skps[0].UnlockConditions.UnlockHash() for _, skp := range skps { if skp.UnlockConditions.UnlockHash() != baseUnlockHash { return ErrInconsistentKeys } } if uint64(len(skps)) < skps[0].UnlockConditions.SignaturesRequired { return ErrInsufficientKeys } // Drop all unneeded keys. skps = skps[0:skps[0].UnlockConditions.SignaturesRequired] // Merge the keys into a single spendableKey and save it to the wallet. var sk spendableKey sk.UnlockConditions = skps[0].UnlockConditions for _, skp := range skps { sk.SecretKeys = append(sk.SecretKeys, skp.SecretKey) } err := w.loadSpendableKey(masterKey, sk) if err != nil { return err } err = w.saveSettingsSync() if err != nil { return err } return w.createBackup(filepath.Join(w.persistDir, "Sia Wallet Encrypted Backup - "+persist.RandomSuffix()+settingsFileSuffix)) } // LoadSiagKeys loads a set of siag-generated keys into the wallet. func (w *Wallet) LoadSiagKeys(masterKey crypto.TwofishKey, keyfiles []string) error { if err := w.tg.Add(); err != nil { return err } defer w.tg.Done() w.mu.Lock() defer w.mu.Unlock() err := w.checkMasterKey(masterKey) if err != nil { return err } return w.loadSiagKeys(masterKey, keyfiles) } // Load033xWallet loads a v0.3.3.x wallet as an unseeded key, such that the // funds become spendable to the current wallet. func (w *Wallet) Load033xWallet(masterKey crypto.TwofishKey, filepath033x string) error { if err := w.tg.Add(); err != nil { return err } defer w.tg.Done() w.mu.Lock() defer w.mu.Unlock() err := w.checkMasterKey(masterKey) if err != nil { return err } var savedKeys []SavedKey033x err = encoding.ReadFile(filepath033x, &savedKeys) if err != nil { return err } var seedsLoaded int for _, savedKey := range savedKeys { spendKey := spendableKey{ UnlockConditions: savedKey.UnlockConditions, SecretKeys: []crypto.SecretKey{savedKey.SecretKey}, } err = w.loadSpendableKey(masterKey, spendKey) if err != nil && err != errDuplicateSpendableKey { return err } if err == nil { seedsLoaded++ } } err = w.saveSettingsSync() if err != nil { return err } if seedsLoaded == 0 { return errAllDuplicates } return w.createBackup(filepath.Join(w.persistDir, "Sia Wallet Encrypted Backup - "+persist.RandomSuffix()+settingsFileSuffix)) } Sia-1.0.4/modules/wallet/unseeded_test.go000066400000000000000000000054241302033070500203500ustar00rootroot00000000000000package wallet import ( "testing" "github.com/NebulousLabs/Sia/types" ) // TestIntegrationLoad1of1Siag loads a 1 of 1 unseeded key generated by siag // and then tries to spend the siafunds contained within. The key is taken from // the testing keys. func TestIntegrationLoad1of1Siag(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestIntegrationLoad1of1Siag") if err != nil { t.Fatal(err) } defer wt.closeWt() // Load the key into the wallet. err = wt.wallet.LoadSiagKeys(wt.walletMasterKey, []string{"../../types/siag0of1of1.siakey"}) if err != nil { t.Error(err) } // Create a second wallet that loads the persist structures of the existing // wallet. This wallet should have a siafund balance. w, err := New(wt.cs, wt.tpool, wt.wallet.persistDir) if err != nil { t.Fatal(err) } defer w.Close() err = w.Unlock(wt.walletMasterKey) if err != nil { t.Fatal(err) } _, siafundBal, _ := w.ConfirmedBalance() if !siafundBal.Equals64(2000) { t.Error("expecting a siafund balance of 2000 from the 1of1 key") } // Send some siafunds to the void. _, err = w.SendSiafunds(types.NewCurrency64(12), types.UnlockHash{}) if err != nil { t.Fatal(err) } _, err = wt.miner.AddBlock() if err != nil { t.Fatal(err) } _, siafundBal, _ = w.ConfirmedBalance() if !siafundBal.Equals64(1988) { t.Error("expecting balance of 1988 after sending siafunds to the void") } } // TestIntegrationLoad2of3Siag loads a 2 of 3 unseeded key generated by siag // and then tries to spend the siafunds contained within. The key is taken from // the testing keys. func TestIntegrationLoad2of3Siag(t *testing.T) { if testing.Short() { t.SkipNow() } wt, err := createWalletTester("TestIntegrationLoad2of3Siag") if err != nil { t.Fatal(err) } defer wt.closeWt() // Load the key into the wallet. err = wt.wallet.LoadSiagKeys(wt.walletMasterKey, []string{"../../types/siag0of2of3.siakey", "../../types/siag1of2of3.siakey"}) if err != nil { t.Error(err) } // Create a second wallet that loads the persist structures of the existing // wallet. This wallet should have a siafund balance. w, err := New(wt.cs, wt.tpool, wt.wallet.persistDir) if err != nil { t.Fatal(err) } defer w.Close() err = w.Unlock(wt.walletMasterKey) if err != nil { t.Fatal(err) } _, siafundBal, _ := w.ConfirmedBalance() if !siafundBal.Equals64(7000) { t.Error("expecting a siafund balance of 7000 from the 2of3 key") } // Send some siafunds to the void. _, err = w.SendSiafunds(types.NewCurrency64(12), types.UnlockHash{}) if err != nil { t.Fatal(err) } _, err = wt.miner.AddBlock() if err != nil { t.Fatal(err) } _, siafundBal, _ = w.ConfirmedBalance() if !siafundBal.Equals64(6988) { t.Error("expecting balance of 6988 after sending siafunds to the void") } } Sia-1.0.4/modules/wallet/update.go000066400000000000000000000224521302033070500167770ustar00rootroot00000000000000package wallet import ( "math" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) // updateConfirmedSet uses a consensus change to update the confirmed set of // outputs as understood by the wallet. func (w *Wallet) updateConfirmedSet(cc modules.ConsensusChange) { for _, diff := range cc.SiacoinOutputDiffs { // Verify that the diff is relevant to the wallet. _, exists := w.keys[diff.SiacoinOutput.UnlockHash] if !exists { continue } _, exists = w.siacoinOutputs[diff.ID] if diff.Direction == modules.DiffApply { if build.DEBUG && exists { panic("adding an existing output to wallet") } w.siacoinOutputs[diff.ID] = diff.SiacoinOutput } else { if build.DEBUG && !exists { panic("deleting nonexisting output from wallet") } delete(w.siacoinOutputs, diff.ID) } } for _, diff := range cc.SiafundOutputDiffs { // Verify that the diff is relevant to the wallet. _, exists := w.keys[diff.SiafundOutput.UnlockHash] if !exists { continue } _, exists = w.siafundOutputs[diff.ID] if diff.Direction == modules.DiffApply { if build.DEBUG && exists { panic("adding an existing output to wallet") } w.siafundOutputs[diff.ID] = diff.SiafundOutput } else { if build.DEBUG && !exists { panic("deleting nonexisting output from wallet") } delete(w.siafundOutputs, diff.ID) } } for _, diff := range cc.SiafundPoolDiffs { if diff.Direction == modules.DiffApply { w.siafundPool = diff.Adjusted } else { w.siafundPool = diff.Previous } } } // revertHistory reverts any transaction history that was destroyed by reverted // blocks in the consensus change. func (w *Wallet) revertHistory(cc modules.ConsensusChange) { for _, block := range cc.RevertedBlocks { // Remove any transactions that have been reverted. for i := len(block.Transactions) - 1; i >= 0; i-- { // If the transaction is relevant to the wallet, it will be the // most recent transaction appended to w.processedTransactions. // Relevance can be determined just by looking at the last element // of w.processedTransactions. txn := block.Transactions[i] txid := txn.ID() if len(w.processedTransactions) > 0 && txid == w.processedTransactions[len(w.processedTransactions)-1].TransactionID { w.processedTransactions = w.processedTransactions[:len(w.processedTransactions)-1] delete(w.processedTransactionMap, txid) } } // Remove the miner payout transaction if applicable. for _, mp := range block.MinerPayouts { _, exists := w.keys[mp.UnlockHash] if exists { w.processedTransactions = w.processedTransactions[:len(w.processedTransactions)-1] delete(w.processedTransactionMap, types.TransactionID(block.ID())) break } } w.consensusSetHeight-- } } // applyHistory applies any transaction history that was introduced by the // applied blocks. func (w *Wallet) applyHistory(cc modules.ConsensusChange) { for _, block := range cc.AppliedBlocks { w.consensusSetHeight++ // Apply the miner payout transaction if applicable. minerPT := modules.ProcessedTransaction{ Transaction: types.Transaction{}, TransactionID: types.TransactionID(block.ID()), ConfirmationHeight: w.consensusSetHeight, ConfirmationTimestamp: block.Timestamp, } relevant := false for i, mp := range block.MinerPayouts { _, exists := w.keys[mp.UnlockHash] if exists { relevant = true } minerPT.Outputs = append(minerPT.Outputs, modules.ProcessedOutput{ FundType: types.SpecifierMinerPayout, MaturityHeight: w.consensusSetHeight + types.MaturityDelay, WalletAddress: exists, RelatedAddress: mp.UnlockHash, Value: mp.Value, }) w.historicOutputs[types.OutputID(block.MinerPayoutID(uint64(i)))] = mp.Value } if relevant { w.processedTransactions = append(w.processedTransactions, minerPT) w.processedTransactionMap[minerPT.TransactionID] = &w.processedTransactions[len(w.processedTransactions)-1] } for _, txn := range block.Transactions { relevant := false pt := modules.ProcessedTransaction{ Transaction: txn, TransactionID: txn.ID(), ConfirmationHeight: w.consensusSetHeight, ConfirmationTimestamp: block.Timestamp, } for _, sci := range txn.SiacoinInputs { _, exists := w.keys[sci.UnlockConditions.UnlockHash()] if exists { relevant = true } pt.Inputs = append(pt.Inputs, modules.ProcessedInput{ FundType: types.SpecifierSiacoinInput, WalletAddress: exists, RelatedAddress: sci.UnlockConditions.UnlockHash(), Value: w.historicOutputs[types.OutputID(sci.ParentID)], }) } for i, sco := range txn.SiacoinOutputs { _, exists := w.keys[sco.UnlockHash] if exists { relevant = true } pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{ FundType: types.SpecifierSiacoinOutput, MaturityHeight: w.consensusSetHeight, WalletAddress: exists, RelatedAddress: sco.UnlockHash, Value: sco.Value, }) w.historicOutputs[types.OutputID(txn.SiacoinOutputID(uint64(i)))] = sco.Value } for _, sfi := range txn.SiafundInputs { _, exists := w.keys[sfi.UnlockConditions.UnlockHash()] if exists { relevant = true } sfiValue := w.historicOutputs[types.OutputID(sfi.ParentID)] pt.Inputs = append(pt.Inputs, modules.ProcessedInput{ FundType: types.SpecifierSiafundInput, WalletAddress: exists, RelatedAddress: sfi.UnlockConditions.UnlockHash(), Value: sfiValue, }) claimValue := w.siafundPool.Sub(w.historicClaimStarts[sfi.ParentID]).Mul(sfiValue) pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{ FundType: types.SpecifierClaimOutput, MaturityHeight: w.consensusSetHeight + types.MaturityDelay, WalletAddress: exists, RelatedAddress: sfi.ClaimUnlockHash, Value: claimValue, }) } for i, sfo := range txn.SiafundOutputs { _, exists := w.keys[sfo.UnlockHash] if exists { relevant = true } pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{ FundType: types.SpecifierSiafundOutput, MaturityHeight: w.consensusSetHeight, WalletAddress: exists, RelatedAddress: sfo.UnlockHash, Value: sfo.Value, }) w.historicOutputs[types.OutputID(txn.SiafundOutputID(uint64(i)))] = sfo.Value w.historicClaimStarts[txn.SiafundOutputID(uint64(i))] = sfo.ClaimStart } for _, fee := range txn.MinerFees { pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{ FundType: types.SpecifierMinerFee, Value: fee, }) } if relevant { w.processedTransactions = append(w.processedTransactions, pt) w.processedTransactionMap[pt.TransactionID] = &w.processedTransactions[len(w.processedTransactions)-1] } } } } // ProcessConsensusChange parses a consensus change to update the set of // confirmed outputs known to the wallet. func (w *Wallet) ProcessConsensusChange(cc modules.ConsensusChange) { if err := w.tg.Add(); err != nil { // The wallet should gracefully reject updates from the consensus set // or transaction pool that are sent after the wallet's Close method // has closed the wallet's ThreadGroup. return } defer w.tg.Done() w.mu.Lock() defer w.mu.Unlock() w.updateConfirmedSet(cc) w.revertHistory(cc) w.applyHistory(cc) } // ReceiveUpdatedUnconfirmedTransactions updates the wallet's unconfirmed // transaction set. func (w *Wallet) ReceiveUpdatedUnconfirmedTransactions(txns []types.Transaction, _ modules.ConsensusChange) { if err := w.tg.Add(); err != nil { // Gracefully reject transactions if the wallet's Close method has // closed the wallet's ThreadGroup already. return } defer w.tg.Done() w.mu.Lock() defer w.mu.Unlock() w.unconfirmedProcessedTransactions = nil for _, txn := range txns { // To save on code complexity, relevancy is determined while building // up the wallet transaction. relevant := false pt := modules.ProcessedTransaction{ Transaction: txn, TransactionID: txn.ID(), ConfirmationHeight: types.BlockHeight(math.MaxUint64), ConfirmationTimestamp: types.Timestamp(math.MaxUint64), } for _, sci := range txn.SiacoinInputs { _, exists := w.keys[sci.UnlockConditions.UnlockHash()] if exists { relevant = true } pt.Inputs = append(pt.Inputs, modules.ProcessedInput{ FundType: types.SpecifierSiacoinInput, WalletAddress: exists, RelatedAddress: sci.UnlockConditions.UnlockHash(), Value: w.historicOutputs[types.OutputID(sci.ParentID)], }) } for i, sco := range txn.SiacoinOutputs { _, exists := w.keys[sco.UnlockHash] if exists { relevant = true } pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{ FundType: types.SpecifierSiacoinOutput, MaturityHeight: types.BlockHeight(math.MaxUint64), WalletAddress: exists, RelatedAddress: sco.UnlockHash, Value: sco.Value, }) w.historicOutputs[types.OutputID(txn.SiacoinOutputID(uint64(i)))] = sco.Value } for _, fee := range txn.MinerFees { pt.Outputs = append(pt.Outputs, modules.ProcessedOutput{ FundType: types.SpecifierMinerFee, Value: fee, }) } if relevant { w.unconfirmedProcessedTransactions = append(w.unconfirmedProcessedTransactions, pt) } } } Sia-1.0.4/modules/wallet/wallet.go000066400000000000000000000154001302033070500170000ustar00rootroot00000000000000package wallet // TODO: Theoretically, the transaction builder in this wallet supports // multisig, but there are no automated tests to verify that. import ( "errors" "fmt" "sort" "sync" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/persist" siasync "github.com/NebulousLabs/Sia/sync" "github.com/NebulousLabs/Sia/types" ) const ( // RespendTimeout records the number of blocks that the wallet will wait // before spending an output that has been spent in the past. If the // transaction spending the output has not made it to the transaction pool // after the limit, the assumption is that it never will. RespendTimeout = 40 ) var ( errNilConsensusSet = errors.New("wallet cannot initialize with a nil consensus set") errNilTpool = errors.New("wallet cannot initialize with a nil transaction pool") ) // spendableKey is a set of secret keys plus the corresponding unlock // conditions. The public key can be derived from the secret key and then // matched to the corresponding public keys in the unlock conditions. All // addresses that are to be used in 'FundSiacoins' or 'FundSiafunds' in the // transaction builder must conform to this form of spendable key. type spendableKey struct { UnlockConditions types.UnlockConditions SecretKeys []crypto.SecretKey } // Wallet is an object that tracks balances, creates keys and addresses, // manages building and sending transactions. type Wallet struct { // unlocked indicates whether the wallet is currently storing secret keys // in memory. subscribed indicates whether the wallet has subscribed to the // consensus set yet - the wallet is unable to subscribe to the consensus // set until it has been unlocked for the first time. The primary seed is // used to generate new addresses for the wallet. unlocked bool subscribed bool persist WalletPersist primarySeed modules.Seed // The wallet's dependencies. The items 'consensusSetHeight' and // 'siafundPool' are tracked separately from the consensus set to minimize // the number of queries that the wallet needs to make to the consensus // set; queries to the consensus set are very slow. cs modules.ConsensusSet tpool modules.TransactionPool consensusSetHeight types.BlockHeight siafundPool types.Currency // The following set of fields are responsible for tracking the confirmed // outputs, and for being able to spend them. The seeds are used to derive // the keys that are tracked on the blockchain. All keys are pregenerated // from the seeds, when checking new outputs or spending outputs, the seeds // are not referenced at all. The seeds are only stored so that the user // may access them. // // siacoinOutptus, siafundOutputs, and spentOutputs are kept so that they // can be scanned when trying to fund transactions. seeds []modules.Seed keys map[types.UnlockHash]spendableKey siacoinOutputs map[types.SiacoinOutputID]types.SiacoinOutput siafundOutputs map[types.SiafundOutputID]types.SiafundOutput spentOutputs map[types.OutputID]types.BlockHeight // The following fields are kept to track transaction history. // processedTransactions are stored in chronological order, and have a map for // constant time random access. The set of full transactions is kept as // well, ordering can be determined by the processedTransactions slice. // // The unconfirmed transactions are kept the same way, except without the // random access. It is assumed that the list of unconfirmed transactions // will be small enough that this will not be a problem. // // historicOutputs is kept so that the values of transaction inputs can be // determined. historicOutputs is never cleared, but in general should be // small compared to the list of transactions. processedTransactions []modules.ProcessedTransaction processedTransactionMap map[types.TransactionID]*modules.ProcessedTransaction unconfirmedProcessedTransactions []modules.ProcessedTransaction // TODO: Storing the whole set of historic outputs is expensive and // unnecessary. There's a better way to do it. historicOutputs map[types.OutputID]types.Currency historicClaimStarts map[types.SiafundOutputID]types.Currency persistDir string log *persist.Logger mu sync.RWMutex // The wallet's ThreadGroup tells tracked functions to shut down and // blocks until they have all exited before returning from Close. tg siasync.ThreadGroup } // New creates a new wallet, loading any known addresses from the input file // name and then using the file to save in the future. Keys and addresses are // not loaded into the wallet during the call to 'new', but rather during the // call to 'Unlock'. func New(cs modules.ConsensusSet, tpool modules.TransactionPool, persistDir string) (*Wallet, error) { // Check for nil dependencies. if cs == nil { return nil, errNilConsensusSet } if tpool == nil { return nil, errNilTpool } // Initialize the data structure. w := &Wallet{ cs: cs, tpool: tpool, keys: make(map[types.UnlockHash]spendableKey), siacoinOutputs: make(map[types.SiacoinOutputID]types.SiacoinOutput), siafundOutputs: make(map[types.SiafundOutputID]types.SiafundOutput), spentOutputs: make(map[types.OutputID]types.BlockHeight), processedTransactionMap: make(map[types.TransactionID]*modules.ProcessedTransaction), historicOutputs: make(map[types.OutputID]types.Currency), historicClaimStarts: make(map[types.SiafundOutputID]types.Currency), persistDir: persistDir, } err := w.initPersist() if err != nil { return nil, err } return w, nil } // Close terminates all ongoing processes involving the wallet, enabling // garbage collection. func (w *Wallet) Close() error { if err := w.tg.Stop(); err != nil { return err } var errs []error // Lock the wallet outside of mu.Lock because Lock uses its own mu.Lock. // Once the wallet is locked it cannot be unlocked except using the // unexported unlock method (w.Unlock returns an error if the wallet's // ThreadGroup is stopped). if w.Unlocked() { if err := w.Lock(); err != nil { errs = append(errs, err) } } w.mu.Lock() defer w.mu.Unlock() w.cs.Unsubscribe(w) w.tpool.Unsubscribe(w) if err := w.log.Close(); err != nil { errs = append(errs, fmt.Errorf("log.Close failed: %v", err)) } return build.JoinErrors(errs, "; ") } // AllAddresses returns all addresses that the wallet is able to spend from, // including unseeded addresses. Addresses are returned sorted in byte-order. func (w *Wallet) AllAddresses() []types.UnlockHash { w.mu.RLock() defer w.mu.RUnlock() addrs := make(types.UnlockHashSlice, 0, len(w.keys)) for addr := range w.keys { addrs = append(addrs, addr) } sort.Sort(addrs) return addrs } Sia-1.0.4/modules/wallet/wallet_test.go000066400000000000000000000133121302033070500200370ustar00rootroot00000000000000package wallet import ( "crypto/rand" "path/filepath" "testing" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/consensus" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/modules/miner" "github.com/NebulousLabs/Sia/modules/transactionpool" "github.com/NebulousLabs/Sia/types" ) // A Wallet tester contains a ConsensusTester and has a bunch of helpful // functions for facilitating wallet integration testing. type walletTester struct { cs modules.ConsensusSet gateway modules.Gateway tpool modules.TransactionPool miner modules.TestMiner wallet *Wallet walletMasterKey crypto.TwofishKey persistDir string } // createWalletTester takes a testing.T and creates a WalletTester. func createWalletTester(name string) (*walletTester, error) { // Create the modules testdir := build.TempDir(modules.WalletDir, name) g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, err } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { return nil, err } w, err := New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } var masterKey crypto.TwofishKey _, err = rand.Read(masterKey[:]) if err != nil { return nil, err } _, err = w.Encrypt(masterKey) if err != nil { return nil, err } err = w.Unlock(masterKey) if err != nil { return nil, err } m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } // Assemble all componenets into a wallet tester. wt := &walletTester{ cs: cs, gateway: g, tpool: tp, miner: m, wallet: w, walletMasterKey: masterKey, persistDir: testdir, } // Mine blocks until there is money in the wallet. for i := types.BlockHeight(0); i <= types.MaturityDelay; i++ { b, _ := wt.miner.FindBlock() err := wt.cs.AcceptBlock(b) if err != nil { return nil, err } } return wt, nil } // createBlankWalletTester creates a wallet tester that has not mined any // blocks or encrypted the wallet. func createBlankWalletTester(name string) (*walletTester, error) { // Create the modules testdir := build.TempDir(modules.WalletDir, name) g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { return nil, err } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { return nil, err } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { return nil, err } w, err := New(cs, tp, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } m, err := miner.New(cs, tp, w, filepath.Join(testdir, modules.WalletDir)) if err != nil { return nil, err } // Assemble all componenets into a wallet tester. wt := &walletTester{ gateway: g, cs: cs, tpool: tp, miner: m, wallet: w, persistDir: testdir, } return wt, nil } // closeWt closes all of the modules in the wallet tester. func (wt *walletTester) closeWt() { errs := []error{ wt.gateway.Close(), wt.cs.Close(), wt.tpool.Close(), wt.miner.Close(), wt.wallet.Close(), } if err := build.JoinErrors(errs, "; "); err != nil { panic(err) } } // TestNilInputs tries starting the wallet using nil inputs. func TestNilInputs(t *testing.T) { testdir := build.TempDir(modules.WalletDir, "TestNilInputs") g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { t.Fatal(err) } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { t.Fatal(err) } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { t.Fatal(err) } wdir := filepath.Join(testdir, modules.WalletDir) _, err = New(cs, nil, wdir) if err != errNilTpool { t.Error(err) } _, err = New(nil, tp, wdir) if err != errNilConsensusSet { t.Error(err) } _, err = New(nil, nil, wdir) if err != errNilConsensusSet { t.Error(err) } } // TestAllAddresses checks that AllAddresses returns all of the wallet's // addresses in sorted order. func TestAllAddresses(t *testing.T) { wt, err := createBlankWalletTester("TestAllAddresses") if err != nil { t.Fatal(err) } defer wt.closeWt() wt.wallet.keys[types.UnlockHash{1}] = spendableKey{} wt.wallet.keys[types.UnlockHash{5}] = spendableKey{} wt.wallet.keys[types.UnlockHash{0}] = spendableKey{} wt.wallet.keys[types.UnlockHash{2}] = spendableKey{} wt.wallet.keys[types.UnlockHash{4}] = spendableKey{} wt.wallet.keys[types.UnlockHash{3}] = spendableKey{} addrs := wt.wallet.AllAddresses() for i := range addrs { if addrs[i][0] != byte(i) { t.Error("address sorting failed:", i, addrs[i][0]) } } } // TestCloseWallet tries to close the wallet. func TestCloseWallet(t *testing.T) { if testing.Short() { t.Skip() } testdir := build.TempDir(modules.WalletDir, "TestCloseWallet") g, err := gateway.New("localhost:0", false, filepath.Join(testdir, modules.GatewayDir)) if err != nil { t.Fatal(err) } cs, err := consensus.New(g, false, filepath.Join(testdir, modules.ConsensusDir)) if err != nil { t.Fatal(err) } tp, err := transactionpool.New(cs, g, filepath.Join(testdir, modules.TransactionPoolDir)) if err != nil { t.Fatal(err) } wdir := filepath.Join(testdir, modules.WalletDir) w, err := New(cs, tp, wdir) if err != nil { t.Fatal(err) } if err := w.Close(); err != nil { t.Fatal(err) } } Sia-1.0.4/persist/000077500000000000000000000000001302033070500137125ustar00rootroot00000000000000Sia-1.0.4/persist/boltdb.go000066400000000000000000000041111302033070500155040ustar00rootroot00000000000000package persist import ( "time" "github.com/NebulousLabs/bolt" ) // BoltDatabase is a persist-level wrapper for the bolt database, providing // extra information such as a version number. type BoltDatabase struct { Metadata *bolt.DB } // checkMetadata confirms that the metadata in the database is // correct. If there is no metadata, correct metadata is inserted func (db *BoltDatabase) checkMetadata(md Metadata) error { err := db.Update(func(tx *bolt.Tx) error { // Check if the database has metadata. If not, create metadata for the // database. bucket := tx.Bucket([]byte("Metadata")) if bucket == nil { err := db.updateMetadata(tx) if err != nil { return err } return nil } // Verify that the metadata matches the expected metadata. header := bucket.Get([]byte("Header")) if string(header) != md.Header { return ErrBadHeader } version := bucket.Get([]byte("Version")) if string(version) != md.Version { return ErrBadVersion } return nil }) return err } // updateMetadata will set the contents of the metadata bucket to the values // in db.Metadata. func (db *BoltDatabase) updateMetadata(tx *bolt.Tx) error { bucket, err := tx.CreateBucketIfNotExists([]byte("Metadata")) if err != nil { return err } err = bucket.Put([]byte("Header"), []byte(db.Header)) if err != nil { return err } err = bucket.Put([]byte("Version"), []byte(db.Version)) if err != nil { return err } return nil } // Close closes the database. func (db *BoltDatabase) Close() error { return db.DB.Close() } // OpenDatabase opens a database and validates its metadata. func OpenDatabase(md Metadata, filename string) (*BoltDatabase, error) { // Open the database using a 3 second timeout (without the timeout, // database will potentially hang indefinitely. db, err := bolt.Open(filename, 0600, &bolt.Options{Timeout: 3 * time.Second}) if err != nil { return nil, err } // Check the metadata. boltDB := &BoltDatabase{ Metadata: md, DB: db, } err = boltDB.checkMetadata(md) if err != nil { db.Close() return nil, err } return boltDB, nil } Sia-1.0.4/persist/boltdb_test.go000066400000000000000000000327431302033070500165570ustar00rootroot00000000000000package persist import ( "math/rand" "os" "path/filepath" "runtime" "testing" "time" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/bolt" ) // testInputs and testFilenames are global variables because most tests require // a variety of metadata and filename inputs (although only TestCheckMetadata // and TestIntegratedCheckMetadata use testInput.newMd and testInput.err). // Weird strings are from https://github.com/minimaxir/big-ltist-of-naughty-strings var ( testInputs = []struct { md Metadata newMd Metadata err error }{ {Metadata{"1sadf23", "12253"}, Metadata{"1sa-df23", "12253"}, ErrBadHeader}, {Metadata{"$@#$%^&", "$@#$%^&"}, Metadata{"$@#$%^&", "$@#$%!^&"}, ErrBadVersion}, {Metadata{"//", "//"}, Metadata{"////", "//"}, ErrBadHeader}, {Metadata{":]", ":)"}, Metadata{":]", ":("}, ErrBadVersion}, {Metadata{"¯|_(ツ)_|¯", "_|¯(ツ)¯|_"}, Metadata{"¯|_(ツ)_|¯", "_|¯(ツ)_|¯"}, ErrBadVersion}, {Metadata{"世界", "怎么办呢"}, Metadata{"世界", "怎么好呢"}, ErrBadVersion}, {Metadata{" ", " "}, Metadata{"\t", " "}, ErrBadHeader}, {Metadata{"", ""}, Metadata{"asdf", ""}, ErrBadHeader}, {Metadata{"", "_"}, Metadata{"", ""}, ErrBadVersion}, {Metadata{"%&*", "#@$"}, Metadata{"", "#@$"}, ErrBadHeader}, {Metadata{"a.sdf", "0.30.2"}, Metadata{"a.sdf", "0.3.02"}, ErrBadVersion}, {Metadata{"/", "/"}, Metadata{"//", "/"}, ErrBadHeader}, {Metadata{"%*.*s", "%d"}, Metadata{"%*.*s", "% d"}, ErrBadVersion}, {Metadata{" ", ""}, Metadata{" ", ""}, ErrBadHeader}, {Metadata{"⒯⒣⒠ ⒬⒰⒤⒞⒦ ⒝⒭⒪⒲⒩ ⒡⒪⒳ ⒥⒰⒨⒫⒮ ⒪⒱⒠⒭ ⒯⒣⒠ ⒧⒜⒵⒴ ⒟⒪⒢", "undefined"}, Metadata{"⒯⒣⒠ ⒬⒰⒤⒞⒦ ⒝⒭⒪⒲⒩ ⒡⒪⒳ ⒥⒰⒨⒫⒮ ⒪⒱⒠⒭ ⒯⒣⒠ ⒧⒜⒵⒴ ⒟⒪⒢", "␢undefined"}, ErrBadVersion}, {Metadata{" ", "  "}, Metadata{"  ", "  "}, ErrBadHeader}, {Metadata{"\xF0\x9F\x98\x8F", "\xF0\x9F\x98\xBE"}, Metadata{"\xF0\x9F\x98\x8F", " \xF0\x9F\x98\xBE"}, ErrBadVersion}, {Metadata{"'", ""}, Metadata{"`", ""}, ErrBadHeader}, {Metadata{"", "-"}, Metadata{"", "-␡"}, ErrBadVersion}, {Metadata{"", "(ﾉಥ益ಥ ┻━┻"}, Metadata{"", "(ﾉ\nಥ益ಥ ┻━┻"}, ErrBadVersion}, {Metadata{"\n\n", "Ṱ̺̺o͞ ̷i̲̬n̝̗v̟̜o̶̙kè͚̮ ̖t̝͕h̼͓e͇̣ ̢̼h͚͎i̦̲v̻͍e̺̭-m̢iͅn̖̺d̵̼ ̞̥r̛̗e͙p͠r̼̞e̺̠s̘͇e͉̥ǹ̬͎t͍̬i̪̱n͠g̴͉ ͏͉c̬̟h͡a̫̻o̫̟s̗̦.̨̹"}, Metadata{"\n\n", "Ṱ̺̺o͞ ̷i̲̬n̝̗v̟̜o̶̙kè͚̮ t̝͕h̼͓e͇̣ ̢̼h͚͎i̦̲v̻͍e̺̭-m̢iͅn̖̺d̵̼ ̞̥r̛̗e͙p͠r̼̞e̺̠s̘͇e͉̥ǹ̬͎t͍̬i̪̱n͠g̴͉ ͏͉c̬̟h͡a̫̻o̫̟s̗̦.̨̹"}, ErrBadVersion}, } testFilenames = []string{ "_", "-", "1234sg", "@#$%@#", "你好好q wgc好", "\xF0\x9F\x99\x8A", "␣", " ", "$HOME", ",.;'[]-=", "%s", } ) // TestOpenDatabase tests calling OpenDatabase on the following types of // database: // - a database that has not yet been created // - an existing empty database // - an existing nonempty database // Along the way, it also tests calling Close on: // - a newly-created database // - a newly-filled database // - a newly-emptied database func TestOpenDatabase(t *testing.T) { if testing.Short() { t.SkipNow() } testBuckets := [][]byte{ []byte("Fake Bucket123!@#$"), []byte("你好好好"), []byte("¯|_(ツ)_|¯"), []byte("Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗"), []byte("﷽"), []byte("(ﾉಥ益ಥ ┻━┻"), []byte("Ṱ̺̺o͞ ̷i̲̬n̝̗v̟̜o̶̙kè͚̮ ̖t̝͕h̼͓e͇̣ ̢̼h͚͎i̦̲v̻͍e̺̭-m̢iͅn̖̺d̵̼ ̞̥r̛̗e͙p͠r̼̞e̺̠s̘͇e͉̥ǹ̬͎t͍̬i̪̱n͠g̴͉ ͏͉c̬̟h͡a̫̻o̫̟s̗̦.̨̹"), []byte("0xbadidea"), []byte("␣"), []byte("你好好好"), } // Create a folder for the database file. If a folder by that name exists // already, it will be replaced by an empty folder. testDir := build.TempDir(persistDir, "TestOpenNewDatabase") err := os.MkdirAll(testDir, 0700) if err != nil { t.Fatal(err) } for i, in := range testInputs { dbFilename := testFilenames[i%len(testFilenames)] dbFilepath := filepath.Join(testDir, dbFilename) // Create a new database. db, err := OpenDatabase(in.md, dbFilepath) if err != nil { t.Errorf("calling OpenDatabase on a new database failed for metadata %v, filename %v; error was %v", in.md, dbFilename, err) continue } // Close the newly-created, empty database. err = db.Close() if err != nil { t.Errorf("closing a newly created database failed for metadata %v, filename %v; error was %v", in.md, dbFilename, err) continue } // Call OpenDatabase again, this time on the existing empty database. db, err = OpenDatabase(in.md, dbFilepath) if err != nil { t.Errorf("calling OpenDatabase on an existing empty database failed for metadata %v, filename %v; error was %v", in.md, dbFilename, err) continue } // Create buckets in the database. err = db.Update(func(tx *bolt.Tx) error { for _, testBucket := range testBuckets { _, err := tx.CreateBucketIfNotExists(testBucket) if err != nil { t.Errorf("db.Update failed on bucket name %v for metadata %v, filename %v; error was %v", testBucket, in.md, dbFilename, err) return err } } return nil }) if err != nil { t.Error(err) continue } // Make sure CreateBucketIfNotExists method handles invalid (nil) // bucket name. err = db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucketIfNotExists(nil) return err }) if err != bolt.ErrBucketNameRequired { } // Fill each bucket with a random number (0-9, inclusive) of key/value // pairs, where each key is a length-10 random byteslice and each value // is a length-1000 random byteslice. err = db.Update(func(tx *bolt.Tx) error { for _, testBucket := range testBuckets { b := tx.Bucket(testBucket) x := rand.Intn(10) for i := 0; i <= x; i++ { k := make([]byte, 10) rand.Read(k) v := make([]byte, 1e3) rand.Read(v) err := b.Put(k, v) if err != nil { t.Errorf("db.Update failed to fill bucket %v for metadata %v, filename %v; error was %v", testBucket, in.md, dbFilename, err) return err } } } return nil }) if err != nil { t.Error(err) continue } // Close the newly-filled database. err = db.Close() if err != nil { t.Errorf("closing a newly-filled database failed for metadata %v, filename %v; error was %v", in.md, dbFilename, err) continue } // Call OpenDatabase on the database now that it's been filled. db, err = OpenDatabase(in.md, dbFilepath) if err != nil { t.Error(err) continue } // Empty every bucket in the database. err = db.Update(func(tx *bolt.Tx) error { for _, testBucket := range testBuckets { b := tx.Bucket(testBucket) err := b.ForEach(func(k, v []byte) error { return b.Delete(k) }) if err != nil { return err } } return nil }) // Close and delete the newly emptied database. err = db.Close() if err != nil { t.Errorf("closing a newly-emptied database for metadata %v, filename %v; error was %v", in.md, dbFilename, err) continue } err = os.Remove(dbFilepath) if err != nil { t.Errorf("removing database file failed for metadata %v, filename %v; error was %v", in.md, dbFilename, err) continue } } } // TestErrPermissionOpenDatabase tests calling OpenDatabase on a database file // with the wrong filemode (< 0600), which should result in an os.ErrPermission // error. func TestErrPermissionOpenDatabase(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("can't reproduce on Windows") } const ( dbHeader = "Fake Header" dbVersion = "0.0.0" dbFilename = "Fake Filename" ) testDir := build.TempDir(persistDir, "TestErrPermissionOpenDatabase") err := os.MkdirAll(testDir, 0700) if err != nil { t.Fatal(err) } dbFilepath := filepath.Join(testDir, dbFilename) badFileModes := []os.FileMode{0000, 0001, 0002, 0003, 0004, 0005, 0010, 0040, 0060, 0105, 0110, 0126, 0130, 0143, 0150, 0166, 0170, 0200, 0313, 0470, 0504, 0560, 0566, 0577} // Make sure OpenDatabase returns a permissions error for each of the modes // in badFileModes. for _, mode := range badFileModes { // Create a file named dbFilename in directory testDir with the wrong // permissions (mode < 0600). _, err := os.OpenFile(dbFilepath, os.O_RDWR|os.O_CREATE, mode) if err != nil { t.Fatal(err) } // OpenDatabase should return a permissions error because the database // mode is less than 0600. _, err = OpenDatabase(Metadata{dbHeader, dbVersion}, dbFilepath) if !os.IsPermission(err) { t.Errorf("OpenDatabase failed to return expected error when called on a database with the wrong permissions (%o instead of >= 0600);\n wanted:\topen %v: permission denied\n got:\t\t%v", mode, dbFilepath, err) } err = os.Remove(dbFilepath) if err != nil { t.Error(err) } } } // TestErrTxNotWritable checks that updateMetadata returns an error when called // from a read-only transaction. func TestErrTxNotWritable(t *testing.T) { if testing.Short() { t.SkipNow() } testDir := build.TempDir(persistDir, "TestErrTxNotWritable") err := os.MkdirAll(testDir, 0700) if err != nil { t.Fatal(err) } for i, in := range testInputs { dbFilename := testFilenames[i%len(testFilenames)] dbFilepath := filepath.Join(testDir, dbFilename) db, err := bolt.Open(dbFilepath, 0600, &bolt.Options{Timeout: 3 * time.Second}) if err != nil { t.Fatal(err) } boltDB := &BoltDatabase{ Metadata: in.md, DB: db, } // Should return an error because updateMetadata is being called from // a read-only transaction. err = db.View(boltDB.updateMetadata) if err != bolt.ErrTxNotWritable { t.Errorf("updateMetadata returned wrong error for input %v, filename %v; expected tx not writable, got %v", in.md, dbFilename, err) } err = boltDB.Close() if err != nil { t.Fatal(err) } err = os.Remove(dbFilepath) if err != nil { t.Fatal(err) } } } // TestErrDatabaseNotOpen tests that checkMetadata returns an error when called // on a BoltDatabase that is closed. func TestErrDatabaseNotOpen(t *testing.T) { if testing.Short() { t.SkipNow() } testDir := build.TempDir(persistDir, "TestErrDatabaseNotOpen") err := os.MkdirAll(testDir, 0700) if err != nil { t.Fatal(err) } dbFilepath := filepath.Join(testDir, "fake_filename") md := Metadata{"Fake Header", "Fake Version"} db, err := bolt.Open(dbFilepath, 0600, &bolt.Options{Timeout: 3 * time.Second}) if err != nil { t.Fatal(err) } boltDB := &BoltDatabase{ Metadata: md, DB: db, } err = boltDB.Close() if err != nil { t.Fatal(err) } // Should return an error since boltDB is closed. err = boltDB.checkMetadata(md) if err != bolt.ErrDatabaseNotOpen { t.Errorf("expected database not open, got %v", err) } err = os.Remove(dbFilepath) if err != nil { t.Error(err) } } // TestErrCheckMetadata tests that checkMetadata returns an error when called // on a BoltDatabase whose metadata has been changed. func TestErrCheckMetadata(t *testing.T) { if testing.Short() { t.SkipNow() } testDir := build.TempDir(persistDir, "TestErrCheckMetadata") err := os.MkdirAll(testDir, 0700) if err != nil { t.Fatal(err) } for i, in := range testInputs { dbFilename := testFilenames[i%len(testFilenames)] dbFilepath := filepath.Join(testDir, dbFilename) db, err := bolt.Open(dbFilepath, 0600, &bolt.Options{Timeout: 3 * time.Second}) if err != nil { t.Fatal(err) } boltDB := &BoltDatabase{ Metadata: in.md, DB: db, } err = db.Update(func(tx *bolt.Tx) error { bucket, err := tx.CreateBucketIfNotExists([]byte("Metadata")) if err != nil { return err } err = bucket.Put([]byte("Header"), []byte(in.newMd.Header)) if err != nil { return err } err = bucket.Put([]byte("Version"), []byte(in.newMd.Version)) if err != nil { return err } return nil }) if err != nil { t.Errorf("Put method failed for input %v, filename %v with error %v", in, dbFilename, err) continue } // Should return an error because boltDB's metadata now differs from // its original metadata. err = (*boltDB).checkMetadata(in.md) if err != in.err { t.Errorf("expected %v, got %v for input %v -> %v", in.err, err, in.md, in.newMd) } err = boltDB.Close() if err != nil { t.Fatal(err) } err = os.Remove(dbFilepath) if err != nil { t.Fatal(err) } } } // TestErrIntegratedCheckMetadata checks that checkMetadata returns an error // within OpenDatabase when OpenDatabase is called on a BoltDatabase that has // already been set up with different metadata. func TestErrIntegratedCheckMetadata(t *testing.T) { if testing.Short() { t.SkipNow() } testDir := build.TempDir(persistDir, "TestErrIntegratedCheckMetadata") err := os.MkdirAll(testDir, 0700) if err != nil { t.Fatal(err) } for i, in := range testInputs { dbFilename := testFilenames[i%len(testFilenames)] dbFilepath := filepath.Join(testDir, dbFilename) boltDB, err := OpenDatabase(in.md, dbFilepath) if err != nil { t.Errorf("OpenDatabase failed on input %v, filename %v; error was %v", in, dbFilename, err) continue } err = boltDB.Close() if err != nil { t.Fatal(err) } // Should return an error because boltDB was set up with metadata in.md, not in.newMd boltDB, err = OpenDatabase(in.newMd, dbFilepath) if err != in.err { t.Errorf("expected error %v for input %v and filename %v; got %v instead", in.err, in, dbFilename, err) } err = os.Remove(dbFilepath) if err != nil { t.Fatal(err) } } } Sia-1.0.4/persist/json.go000066400000000000000000000034571302033070500152230ustar00rootroot00000000000000package persist import ( "encoding/json" "io" "os" ) // Load loads json data from a reader. func Load(meta Metadata, data interface{}, r io.Reader) error { var header, version string dec := json.NewDecoder(r) if err := dec.Decode(&header); err != nil { return err } if header != meta.Header { return ErrBadHeader } if err := dec.Decode(&version); err != nil { return err } if version != meta.Version { return ErrBadVersion } if err := dec.Decode(data); err != nil { return err } return nil } // LoadFile loads json data from a file. func LoadFile(meta Metadata, data interface{}, filename string) error { file, err := os.Open(filename) if err != nil { return err } defer file.Close() err = Load(meta, data, file) if err != nil { return err } return nil } // Save saves json data to a writer. func Save(meta Metadata, data interface{}, w io.Writer) error { b, err := json.MarshalIndent(data, "", "\t") if err != nil { return err } enc := json.NewEncoder(w) if err := enc.Encode(meta.Header); err != nil { return err } if err := enc.Encode(meta.Version); err != nil { return err } if _, err = w.Write(b); err != nil { return err } return nil } // SaveFile atomically saves json data to a file. func SaveFile(meta Metadata, data interface{}, filename string) error { file, err := NewSafeFile(filename) if err != nil { return err } defer file.Close() err = Save(meta, data, file) if err != nil { return err } return file.Commit() } // SaveFileSync atomically saves json data to a file and then syncs to disk. func SaveFileSync(meta Metadata, data interface{}, filename string) error { file, err := NewSafeFile(filename) if err != nil { return err } defer file.Close() err = Save(meta, data, file) if err != nil { return err } return file.CommitSync() } Sia-1.0.4/persist/json_test.go000066400000000000000000000052011302033070500162470ustar00rootroot00000000000000package persist import ( "bytes" "os" "testing" "github.com/NebulousLabs/Sia/build" ) // TestSaveLoad checks that saving and loading data behaves as expected. func TestSaveLoad(t *testing.T) { var meta = Metadata{"TestSaveLoad", "0.1"} var saveData int = 3 buf := new(bytes.Buffer) // save data to buffer err := Save(meta, saveData, buf) if err != nil { t.Fatal(err) } data := buf.Bytes() // load valid data var loadData int err = Load(meta, &loadData, bytes.NewReader(data)) if err != nil { t.Fatal(err) } if loadData != saveData { t.Fatalf("loaded data (%v) does not match saved data (%v)", loadData, saveData) } // load with bad metadata err = Load(Metadata{"BadTestSaveLoad", "0.1"}, &loadData, bytes.NewReader(data)) if err != ErrBadHeader { t.Fatal("expected ErrBadHeader, got", err) } err = Load(Metadata{"TestSaveLoad", "-1"}, &loadData, bytes.NewReader(data)) if err != ErrBadVersion { t.Fatal("expected ErrBadVersion, got", err) } // corrupt data, moving back to front data[21] = '}' err = Load(meta, &loadData, bytes.NewReader(data)) if err == nil { t.Fatal("expected error when loading corrupted data") } data[14] = '}' err = Load(meta, &loadData, bytes.NewReader(data)) if err == nil { t.Fatal("expected error when loading corrupted data") } data[0] = '}' err = Load(meta, &loadData, bytes.NewReader(data)) if err == nil { t.Fatal("expected error when loading corrupted data") } } // TestSaveLoadFile tests that saving and loading a file without fsync properly // stores and fetches data. func TestSaveLoadFile(t *testing.T) { var meta = Metadata{"TestSaveLoadFile", "0.1"} var saveData int = 3 os.MkdirAll(build.TempDir("persist"), 0777) filename := build.TempDir("persist", "TestSaveLoadFile") err := SaveFile(meta, saveData, filename) if err != nil { t.Fatal(err) } var loadData int err = LoadFile(meta, &loadData, filename) if err != nil { t.Fatal(err) } if loadData != saveData { t.Fatalf("loaded data (%v) does not match saved data (%v)", loadData, saveData) } } // TestSaveLoadFileSync test that saving and loading a file with fsync properly // stores and fetches data. func TestSaveLoadFileSync(t *testing.T) { var meta = Metadata{"TestSaveLoadFileFsync", "0.1"} var saveData int = 3 os.MkdirAll(build.TempDir("persist"), 0777) filename := build.TempDir("persist", "TestSaveLoadFile") err := SaveFileSync(meta, saveData, filename) if err != nil { t.Fatal(err) } var loadData int err = LoadFile(meta, &loadData, filename) if err != nil { t.Fatal(err) } if loadData != saveData { t.Fatalf("loaded data (%v) does not match saved data (%v)", loadData, saveData) } } Sia-1.0.4/persist/log.go000066400000000000000000000074441302033070500150330ustar00rootroot00000000000000package persist import ( "fmt" "io" "log" "os" "sync" "github.com/NebulousLabs/Sia/build" ) // Logger is a wrapper for the standard library logger that enforces logging // with the Sia-standard settings. It also supports a Close method, which // attempts to close the underlying io.Writer. type Logger struct { *log.Logger w io.Writer } // Close logs a shutdown message and closes the Logger's underlying io.Writer, // if it is also an io.Closer. func (l *Logger) Close() error { l.Output(2, "SHUTDOWN: Logging has terminated.") if c, ok := l.w.(io.Closer); ok { return c.Close() } return nil } // Critical logs a message with a CRITICAL prefix that guides the user to the // Sia github tracker. If debug mode is enabled, it will also write the message // to os.Stderr and panic. Critical should only be called if there has been a // developer error, otherwise Severe should be called. func (l *Logger) Critical(v ...interface{}) { l.Output(2, "CRITICAL: "+fmt.Sprintln(v...)) build.Critical(v...) } // Debug is equivalent to Logger.Print when build.DEBUG is true. Otherwise it // is a no-op. func (l *Logger) Debug(v ...interface{}) { if build.DEBUG { l.Output(2, fmt.Sprint(v...)) } } // Debugf is equivalent to Logger.Printf when build.DEBUG is true. Otherwise it // is a no-op. func (l *Logger) Debugf(format string, v ...interface{}) { if build.DEBUG { l.Output(2, fmt.Sprintf(format, v...)) } } // Debugln is equivalent to Logger.Println when build.DEBUG is true. Otherwise // it is a no-op. func (l *Logger) Debugln(v ...interface{}) { if build.DEBUG { l.Output(2, "[DEBUG] "+fmt.Sprintln(v...)) } } // Severe logs a message with a SEVERE prefix. If debug mode is enabled, it // will also write the message to os.Stderr and panic. Severe should be called // if there is a severe problem with the user's machine or setup that should be // addressed ASAP but does not necessarily require that the machine crash or // exit. func (l *Logger) Severe(v ...interface{}) { l.Output(2, "SEVERE: "+fmt.Sprintln(v...)) build.Severe(v...) } // NewLogger returns a logger that can be closed. Calls should not be made to // the logger after 'Close' has been called. func NewLogger(w io.Writer) *Logger { l := log.New(w, "", log.Ldate|log.Ltime|log.Lmicroseconds|log.Lshortfile|log.LUTC) l.Output(3, "STARTUP: Logging has started.") // Call depth is 3 because NewLogger is usually called by NewFileLogger return &Logger{l, w} } // closeableFile wraps an os.File to perform sanity checks on its Write and // Close methods. When the checks are enabled, calls to Write or Close will // panic if they are called after the file has already been closed. type closeableFile struct { *os.File closed bool mu sync.RWMutex } // Close closes the file and sets the closed flag. func (cf *closeableFile) Close() error { cf.mu.Lock() defer cf.mu.Unlock() // Sanity check - close should not have been called yet. if cf.closed { build.Critical("cannot close the file; already closed") } // Ensure that all data has actually hit the disk. if err := cf.Sync(); err != nil { return err } cf.closed = true return cf.File.Close() } // Write takes the input data and writes it to the file. func (cf *closeableFile) Write(b []byte) (int, error) { cf.mu.RLock() defer cf.mu.RUnlock() // Sanity check - close should not have been called yet. if cf.closed { build.Critical("cannot write to the file after it has been closed") } return cf.File.Write(b) } // NewFileLogger returns a logger that logs to logFilename. The file is opened // in append mode, and created if it does not exist. func NewFileLogger(logFilename string) (*Logger, error) { logFile, err := os.OpenFile(logFilename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0660) if err != nil { return nil, err } cf := &closeableFile{File: logFile} return NewLogger(cf), nil } Sia-1.0.4/persist/log_test.go000066400000000000000000000041741302033070500160670ustar00rootroot00000000000000package persist import ( "io/ioutil" "os" "path/filepath" "strings" "testing" "github.com/NebulousLabs/Sia/build" ) // TestLogger checks that the basic functions of the file logger work as // designed. func TestLogger(t *testing.T) { // Create a folder for the log file. testdir := build.TempDir(persistDir, "TestLogger") err := os.MkdirAll(testdir, 0700) if err != nil { t.Fatal(err) } // Create the logger. logFilename := filepath.Join(testdir, "test.log") fl, err := NewFileLogger(logFilename) if err != nil { t.Fatal(err) } // Write an example statement, and then close the logger. fl.Println("TEST: this should get written to the logfile") err = fl.Close() if err != nil { t.Fatal(err) } // Check that data was written to the log file. There should be three // lines, one for startup, the example line, and one to close the logger. expectedSubstring := []string{"STARTUP", "TEST", "SHUTDOWN", ""} // file ends with a newline fileData, err := ioutil.ReadFile(logFilename) if err != nil { t.Fatal(err) } fileLines := strings.Split(string(fileData), "\n") for i, line := range fileLines { if !strings.Contains(string(line), expectedSubstring[i]) { t.Error("did not find the expected message in the logger") } } if len(fileLines) != 4 { // file ends with a newline t.Error("logger did not create the correct number of lines:", len(fileLines)) } } // TestLoggerCritical prints a critical message from the logger. func TestLoggerCritical(t *testing.T) { // Create a folder for the log file. testdir := build.TempDir(persistDir, "TestLoggerCritical") err := os.MkdirAll(testdir, 0700) if err != nil { t.Fatal(err) } // Create the logger. logFilename := filepath.Join(testdir, "test.log") fl, err := NewFileLogger(logFilename) if err != nil { t.Fatal(err) } // Write a catch for a panic that should trigger when logger.Critical is // called. defer func() { r := recover() if r == nil { t.Error("critical message was not thrown in a panic") } // Close the file logger to clean up the test. err = fl.Close() if err != nil { t.Fatal(err) } }() fl.Critical("a critical message") } Sia-1.0.4/persist/persist.go000066400000000000000000000047711302033070500157430ustar00rootroot00000000000000package persist import ( "crypto/rand" "encoding/base32" "errors" "os" "path/filepath" ) const ( // persistDir defines the folder that is used for testing the persist // package. persistDir = "persist" ) var ( // ErrBadVersion indicates that the version number of the file is not // compatible with the current codebase. ErrBadVersion = errors.New("incompatible version") // ErrBadHeader indicates that the file opened is not the file that was // expected. ErrBadHeader = errors.New("wrong header") ) // Metadata contains the header and version of the data being stored. type Metadata struct { Header, Version string } // RandomSuffix returns a 20 character base32 suffix for a filename. There are // 100 bits of entropy, and a very low probability of colliding with existing // files unintentionally. func RandomSuffix() string { randBytes := make([]byte, 20) rand.Read(randBytes) str := base32.StdEncoding.EncodeToString(randBytes) return str[:20] } // A safeFile is a file that is stored under a temporary filename. When Commit // is called, the file is renamed to its "final" filename. This allows for // atomic updating of files; otherwise, an unexpected shutdown could leave a // valuable file in a corrupted state. Callers must still Close the file handle // as usual. type safeFile struct { *os.File finalName string } // Commit closes the file, and then renames it to the intended final filename. // Commit should not be called from a defer if the function it is being called // from can return an error. func (sf *safeFile) Commit() error { if err := sf.Close(); err != nil { return err } return os.Rename(sf.finalName+"_temp", sf.finalName) } // CommitSync syncs the file, closes it, and then renames it to the intended // final filename. CommitSync should not be called from a defer if the // function it is being called from can return an error. func (sf *safeFile) CommitSync() error { if err := sf.Sync(); err != nil { return err } return sf.Commit() } // NewSafeFile returns a file that can atomically be written to disk, // minimizing the risk of corruption. func NewSafeFile(filename string) (*safeFile, error) { file, err := os.Create(filename + "_temp") if err != nil { return nil, err } // Get the absolute path of the filename so that calling os.Chdir in // between calling NewSafeFile and calling safeFile.Commit does not change // the final file path. absFilename, err := filepath.Abs(filename) if err != nil { return nil, err } return &safeFile{file, absFilename}, nil } Sia-1.0.4/persist/persist_test.go000066400000000000000000000066521302033070500170020ustar00rootroot00000000000000package persist import ( "bytes" "crypto/rand" "io/ioutil" "os" "path/filepath" "testing" "github.com/NebulousLabs/Sia/build" ) // TestIntegrationRandomSuffix checks that the random suffix creator creates // valid files. func TestIntegrationRandomSuffix(t *testing.T) { tmpDir := build.TempDir(persistDir, "TestIntegrationRandomSuffix") err := os.MkdirAll(tmpDir, 0700) if err != nil { t.Fatal(err) } for i := 0; i < 100; i++ { suffix := RandomSuffix() filename := filepath.Join(tmpDir, "test file - "+suffix+".nil") file, err := os.Create(filename) if err != nil { t.Fatal(err) } file.Close() } } // TestAbsolutePathSafeFile tests creating and committing safe files with // absolute paths. func TestAbsolutePathSafeFile(t *testing.T) { tmpDir := build.TempDir(persistDir, "TestAbsolutePathSafeFile") err := os.MkdirAll(tmpDir, 0700) if err != nil { t.Fatal(err) } absPath := filepath.Join(tmpDir, "test") // Create safe file. sf, err := NewSafeFile(absPath) defer sf.Close() if err != nil { t.Fatal(err) } // Check that the name of the file is not equal to the final name of the // file. if sf.Name() == absPath { t.Errorf("safeFile created with filename: %s has temporary filename that is equivalent to finalName: %s\n", absPath, sf.Name()) } // Write random data to the file and commit. data := make([]byte, 10) rand.Read(data) _, err = sf.Write(data) if err != nil { t.Fatal(err) } err = sf.Commit() if err != nil { t.Fatal(err) } // Check that the file exists and has same data that was written to it. dataRead, err := ioutil.ReadFile(absPath) if err != nil { t.Fatal(err) } if !bytes.Equal(data, dataRead) { t.Fatalf("Committed file has different data than was written to it: expected %v, got %v\n", data, dataRead) } } // TestRelativePathSafeFile tests creating and committing safe files with // relative paths. Specifically, we test that calling os.Chdir between creating // and committing a safe file doesn't affect the safe file's final path. The // relative path tested is relative to the working directory. func TestRelativePathSafeFile(t *testing.T) { tmpDir := build.TempDir(persistDir, "TestRelativePathSafeFile") err := os.MkdirAll(tmpDir, 0700) if err != nil { t.Fatal(err) } absPath := filepath.Join(tmpDir, "test") wd, err := os.Getwd() if err != nil { t.Fatal(err) } relPath, err := filepath.Rel(wd, absPath) // Create safe file. sf, err := NewSafeFile(relPath) defer sf.Close() if err != nil { t.Fatal(err) } // Check that the path of the file is not equal to the final path of the // file. if sf.Name() == absPath { t.Errorf("safeFile created with filename: %s has temporary filename that is equivalent to finalName: %s\n", absPath, sf.Name()) } // Write random data to the file. data := make([]byte, 10) rand.Read(data) _, err = sf.Write(data) if err != nil { t.Fatal(err) } // Change directories and commit. tmpChdir := build.TempDir(persistDir, "TestRelativePathSafeFileTmpChdir") err = os.MkdirAll(tmpChdir, 0700) if err != nil { t.Fatal(err) } os.Chdir(tmpChdir) defer os.Chdir(wd) err = sf.CommitSync() if err != nil { t.Fatal(err) } // Check that the file exists and has same data that was written to it. dataRead, err := ioutil.ReadFile(absPath) if err != nil { t.Fatal(err) } if !bytes.Equal(data, dataRead) { t.Fatalf("Committed file has different data than was written to it: expected %v, got %v\n", data, dataRead) } } Sia-1.0.4/profile/000077500000000000000000000000001302033070500136615ustar00rootroot00000000000000Sia-1.0.4/profile/profile.go000066400000000000000000000063411302033070500156540ustar00rootroot00000000000000package profile import ( "errors" "fmt" "os" "path/filepath" "runtime" "runtime/pprof" "sync" "time" "github.com/NebulousLabs/Sia/persist" ) // There's a global lock on cpu and memory profiling, because I'm not sure what // happens if multiple threads call each at the same time. This lock might be // unnecessary. var ( cpuActive bool cpuLock sync.Mutex memActive bool memLock sync.Mutex ) // startCPUProfile starts cpu profiling. An error will be returned if a cpu // profiler is already running. func StartCPUProfile(profileDir, identifier string) error { // Lock the cpu profile lock so that only one profiler is running at a // time. cpuLock.Lock() if cpuActive { cpuLock.Unlock() return errors.New("cannot start cpu profilier, a profiler is already running") } cpuActive = true cpuLock.Unlock() // Start profiling into the profile dir, using the identifer. The timestamp // of the start time of the profiling will be included in the filename. cpuProfileFile, err := os.Create(filepath.Join(profileDir, "cpu-profile-"+identifier+"-"+time.Now().Format(time.RFC3339Nano)+".prof")) if err != nil { return err } pprof.StartCPUProfile(cpuProfileFile) return nil } // stopCPUProfile stops cpu profiling. func StopCPUProfile() { cpuLock.Lock() if cpuActive { pprof.StopCPUProfile() cpuActive = false } cpuLock.Unlock() } // saveMemProfile saves the current memory structure of the program. An error // will be returned if memory profiling is already in progress. Unlike for cpu // profiling, there is no 'stopMemProfile' call - everything happens at once. func SaveMemProfile(profileDir, identifier string) error { memLock.Lock() if memActive { memLock.Unlock() return errors.New("cannot start memory profiler, a memory profiler is already running") } memActive = true memLock.Unlock() // Save the memory profile. memFile, err := os.Create(filepath.Join(profileDir, "mem-profile-"+identifier+"-"+time.Now().Format(time.RFC3339Nano)+".prof")) if err != nil { return err } pprof.WriteHeapProfile(memFile) memLock.Lock() memActive = false memLock.Unlock() return nil } // StartContinuousProfiling will continuously print statistics about the cpu // usage, memory usage, and runtime stats of the program. func StartContinuousProfile(profileDir string) { // Create the folder for all of the profiling results. err := os.MkdirAll(profileDir, 0700) if err != nil { fmt.Println(err) return } // Continuously log statistics about the running Sia application. go func() { // Create the logger. log, err := persist.NewFileLogger(filepath.Join(profileDir, "continuousProfiling.log")) if err != nil { fmt.Println("Profile logging failed:", err) return } // Collect statistics in an infinite loop. sleepTime := time.Second * 20 for { // Sleep for an exponential amount of time each iteration, this // keeps the size of the log small while still providing lots of // information. time.Sleep(sleepTime) sleepTime = time.Duration(1.5 * float64(sleepTime)) var m runtime.MemStats runtime.ReadMemStats(&m) log.Printf("\n\tGoroutines: %v\n\tAlloc: %v\n\tTotalAlloc: %v\n\tHeapAlloc: %v\n\tHeapSys: %v\n", runtime.NumGoroutine(), m.Alloc, m.TotalAlloc, m.HeapAlloc, m.HeapSys) } }() } Sia-1.0.4/profile/server.go000066400000000000000000000002201302033070500155100ustar00rootroot00000000000000// +build profile package profile import ( "net/http" _ "net/http/pprof" ) func init() { go http.ListenAndServe("localhost:10501", nil) } Sia-1.0.4/profile/timing.go000066400000000000000000000016431302033070500155030ustar00rootroot00000000000000package profile import ( "fmt" "time" ) var ( uptime int64 times = make(map[string]int64) activeTimers = make(map[string]int64) ) // Uptime() returns the number of nanoseconds that have passed since the first // call to uptime. func Uptime() int64 { if uptime == 0 { uptime = time.Now().UnixNano() return 0 } return (time.Now().UnixNano() - uptime) / 1e6 } // PrintTimes prints how much time has passed at each timer. func PrintTimes() string { s := "Printing Timers:\n" for name, time := range times { s += fmt.Sprintf("\t%v: %v\n", name, time/1e6) } return s } // ToggleTimer actives a timer known by a given string. If the timer does not // yet exist, it is created. func ToggleTimer(s string) { toggleTime, exists := activeTimers[s] if exists { times[s] = times[s] + (time.Now().UnixNano() - toggleTime) delete(activeTimers, s) } else { activeTimers[s] = time.Now().UnixNano() } } Sia-1.0.4/release.sh000077500000000000000000000025141302033070500142020ustar00rootroot00000000000000#!/bin/bash set -e # version and keys are supplied as arguments version="$1" keyfile="$2" pubkeyfile="$3" # optional if [[ -z $version || -z $keyfile ]]; then echo "Usage: $0 VERSION KEYFILE" exit 1 fi if [[ -z $pubkeyfile ]]; then echo "Warning: no public keyfile supplied. Binaries will not be verified." fi # check for keyfile before proceeding if [ ! -f $keyfile ]; then echo "Key file not found: $keyfile" exit 1 fi keysum=$(shasum -a 256 $keyfile | cut -c -64) if [ $keysum != "735320b4698010500d230c487e970e12776e88f33ad777ab380a493691dadb1b" ]; then echo "Wrong key file: checksum does not match developer key file." exit 1 fi for os in darwin linux windows; do echo Packaging ${os}... # create workspace folder=release/Sia-$version-$os-amd64 rm -rf $folder mkdir -p $folder # compile and sign binaries for pkg in siac siad; do bin=$pkg if [ "$os" == "windows" ]; then bin=${pkg}.exe fi GOOS=${os} go build -o $folder/$bin ./$pkg openssl dgst -sha256 -sign $keyfile -out $folder/${bin}.sig $folder/$bin # verify signature if [[ -n $pubkeyfile ]]; then openssl dgst -sha256 -verify $pubkeyfile -signature $folder/${bin}.sig $folder/$bin fi done # add other artifacts cp -r doc LICENSE README.md $folder # zip ( cd release zip -rq Sia-$version-$os-amd64.zip Sia-$version-$os-amd64 ) done Sia-1.0.4/siac/000077500000000000000000000000001302033070500131405ustar00rootroot00000000000000Sia-1.0.4/siac/README.md000066400000000000000000000177661302033070500144400ustar00rootroot00000000000000Siac Usage ========== `siac` is the command line interface to Sia, for use by power users and those on headless servers. It comes as a part of the command line package, and can be run as `./siac` from the same folder, or just by calling `siac` if you move the binary into your path. Most of the following commands have online help. For example, executing `siac wallet send help` will list the arguments for that command, while `siac host help` will list the commands that can be called pertaining to hosting. `siac help` will list all of the top level command groups that can be used. You can change the address of where siad is pointing using the `-a` flag. For example, `siac -a :9000 status` will display the status of the siad instance launched on the local machine with `siad -a :9000`. Common tasks ------------ * `siac status` view block height Wallet: * `siac wallet init [-p]` initilize a wallet * `siac wallet unlock` unlock a wallet * `siac wallet status` retrieve wallet balance * `siac wallet address` get a wallet address * `siac wallet send [amount] [dest]` sends siacoin to an address Renter: * `siac renter list` list all renter files * `siac renter upload [filepath] [nickname]` upload a file * `siac renter download [nickname] [filepath]` download a file * `siac renter share [nickname] [filepath]` create a .sia file * `siac renter load [filepath]` load a .sia file Full Descriptions ----------------- #### Wallet tasks * `siac wallet init [-p]` encrypts and initializes the wallet. If the `-p` flag is provided, an encryption password is requested from the user. Otherwise the initial seed is used as the encryption password. The wallet must be initialized and unlocked before any actions can be performed on the wallet. Examples: ```bash user@hostname:~$ siac -a :9920 wallet init Seed is: cider sailor incur sober feast unhappy mundane sadness hinder aglow imitate amaze duties arrow gigantic uttered inflamed girth myriad jittery hexagon nail lush reef sushi pastry southern inkling acquire Wallet encrypted with password: cider sailor incur sober feast unhappy mundane sadness hinder aglow imitate amaze duties arrow gigantic uttered inflamed girth myriad jittery hexagon nail lush reef sushi pastry southern inkling acquire ``` ```bash user@hostname:~$ siac -a :9920 wallet init -p Wallet password: Seed is: potato haunted fuming lordship library vane fever powder zippers fabrics dexterity hoisting emails pebbles each vampire rockets irony summon sailor lemon vipers foxes oneself glide cylinder vehicle mews acoustic Wallet encrypted with given password ``` * `siac wallet unlock` prompts the user for the encryption password to the wallet, supplied by the `init` command. The wallet must be initialized and unlocked before any actions can take place. * `siac wallet status` prints information about your wallet. Example: ```bash user@hostname:~$ siac wallet status Wallet status: Encrypted, Unlocked Confirmed Balance: 61516458.00 SC Unconfirmed Balance: 64516461.00 SC Exact: 61516457999999999999999999999999 H ``` * `siac wallet address` returns a never seen before address for sending siacoins to. * `siac wallet send [amount] [dest]` Sends `amount` siacoins to `dest`. `amount` is in the form XXXXUU where an X is a number and U is a unit, for example MS, S, mS, ps, etc. If no unit is given hastings is assumed. `dest` must be a valid siacoin address. * `siac wallet lock` locks a wallet. After calling, the wallet must be unlocked using the encryption password in order to use it further * `siac wallet seeds` returns the list of secret seeds in use by the wallet. These can be used to regenerate the wallet * `siac wallet addseed` prompts the user for his encryption password, as well as a new secret seed. The wallet will then incorporate this seed into itself. This can be used for wallet recovery and merging. #### Host tasks * `host config [setting] [value]` is used to configure hosting. | Setting | Value | | ------------ | ------------------------------------------------ | | totalstorage | The total size you will be hosting from in bytes | | minfilesize | The minimum file size you can host in bytes | | maxfilesize | The maximum file size you can host in bytes | | minduration | The smallest duration you can host for in blocks | | maxduration | The largest duration you can host for in blocks | | price | Number of Siacoins per Gigabyte per month. | You can call this many times to configure you host before announcing. Alternatively, you can manually adjust these parameters inside the `host/config.json` file. * `siac host announce` makes an host announcement. You may optionally supply a specific address to be announced; this allows you to announce a domain name. Announcing a second time after changing settings is not necessary, as the announcement only contains enough information to reach your host. * `siac host status` outputs some of your hosting settings. Example: ```bash user@hostname:~$ siac host status Host settings: Storage: 2.0000 TB (1.524 GB used) Price: 0.000 SC per GB per month Collateral: 0 Max Filesize: 10000000000 Max Duration: 8640 Contracts: 32 ``` * `siac host hostdb` prints a list of all the know active hosts on the network. It can also be called through `siac hostdb` #### Renter tasks * `siac renter upload [filename] [nickname]` uploads a file to the sia network. `filename` is the path to the file you want to upload, and nickname is what you will use to refer to that file in the network. For example, it is common to have the nickname be the same as the filename. * `siac renter list` displays a list of the your uploaded files currently on the sia network by nickname, and their filesizes. * `siac renter download [nickname] [destination]` downloads a file from the sia network onto your computer. `nickname` is the name used to refer to your file in the sia network, and `destination` is the path to where the file will be. If a file already exists there, it will be overwritten. * `siac renter rename [nickname] [newname]` changes the nickname of a file. * `siac renter share [nickname] [filepath]` writes a .sia file pointing to the file specified by `nickname` on the network. The file is written to `filepath`. Note that the `.sia` extension will not be automatically added, and must be part of the path. * `siac renter shareascii [nickname]` writes the .sia file specified by `nickname` to stdout base64 encoded. * `siac renter load [filename]` parses the .sia file at `filename` and adds it to the renters collection of files, so that it can be downloaded. * `siac renter loadascii [data]` parses the siafile passed as an argument and adds it to your collection of files for download. Data will be a very large field. * `siac renter delete [nickname]` removes a file from your list of stored files. This does not remove it from the network, but only from your saved list. * `siac renter queue` shows the download queue. This is only relevant if you have multiple downloads happening simultaneously. #### Gateway tasks * `siac gateway` prints info about the gateway, including its address and how many peers it's connected to. * `siac gateway list` prints a list of all currently connected peers. * `siac gateway connect [address:port]` manually connects to a peer and adds it to the gateway's node list. * `siac gateway disconnect [address:port]` manually disconnects from a peer, but leaves it in the gateway's node list. #### Miner tasks * `siac miner status` returns information about the miner. It is only valid for when siad is running. * `siac miner start` starts running the CPU miner on one thread. This is virtually useless outside of debugging. * `siac miner stop` halts the CPU miner. #### General commands * `siac status` prints the current block ID, current block height, and current target. * `siac stop` sends the stop signal to siad to safely terminate. This has the same affect as C^c on the terminal. * `siac version` displays the version string of siac. * `siac update` checks the server for updates. Sia-1.0.4/siac/consensuscmd.go000066400000000000000000000031021302033070500161670ustar00rootroot00000000000000package main import ( "fmt" "time" "github.com/spf13/cobra" "github.com/NebulousLabs/Sia/api" "github.com/NebulousLabs/Sia/types" ) var ( consensusCmd = &cobra.Command{ Use: "consensus", Short: "Print the current state of consensus", Long: "Print the current state of consensus such as current block, block height, and target.", Run: wrap(consensuscmd), } ) // consensuscmd is the handler for the command `siac consensus`. // Prints the current state of consensus. func consensuscmd() { var cg api.ConsensusGET err := getAPI("/consensus", &cg) if err != nil { die("Could not get current consensus state:", err) } if cg.Synced { fmt.Printf(`Synced: %v Block: %v Height: %v Target: %v `, yesNo(cg.Synced), cg.CurrentBlock, cg.Height, cg.Target) } else { estimatedHeight := estimatedHeightAt(time.Now()) estimatedProgress := float64(cg.Height) / float64(estimatedHeight) * 100 if estimatedProgress > 99 { estimatedProgress = 99 } fmt.Printf(`Synced: %v Height: %v Progress (estimated): %.f%% `, yesNo(cg.Synced), cg.Height, estimatedProgress) } } // estimatedHeightAt returns the estimated block height for the given time. // Block height is estimated by calculating the minutes since a known block in // the past and dividing by 10 minutes (the block time). func estimatedHeightAt(t time.Time) types.BlockHeight { block5e4Timestamp := time.Date(2016, time.May, 11, 19, 33, 0, 0, time.UTC) diff := t.Sub(block5e4Timestamp) estimatedHeight := 5e4 + (diff.Minutes() / 10) return types.BlockHeight(estimatedHeight + 0.5) // round to the nearest block } Sia-1.0.4/siac/consensuscmd_test.go000066400000000000000000000020601302033070500172300ustar00rootroot00000000000000package main import ( "testing" "time" "github.com/NebulousLabs/Sia/types" ) // TestEstimatedHeightAt tests that the expectedHeightAt function correctly // estimates the blockheight (and rounds to the nearest block). func TestEstimatedHeightAt(t *testing.T) { tests := []struct { t time.Time expectedHeight types.BlockHeight }{ // Test on the same block that is used to estimate the height { time.Date(2016, time.May, 11, 19, 33, 0, 0, time.UTC), 5e4, }, // 4 minutes later { time.Date(2016, time.May, 11, 19, 37, 0, 0, time.UTC), 5e4, }, // 5 minutes later { time.Date(2016, time.May, 11, 19, 38, 0, 0, time.UTC), 5e4 + 1, }, // 10 minutes later { time.Date(2016, time.May, 11, 19, 43, 0, 0, time.UTC), 5e4 + 1, }, // 1 day later { time.Date(2016, time.May, 12, 19, 33, 0, 0, time.UTC), 5e4 + 144, }, } for _, tt := range tests { h := estimatedHeightAt(tt.t) if h != tt.expectedHeight { t.Errorf("expected an estimated height of %v, but got %v", tt.expectedHeight, h) } } } Sia-1.0.4/siac/daemoncmd.go000066400000000000000000000042121302033070500154150ustar00rootroot00000000000000package main import ( "fmt" "github.com/NebulousLabs/Sia/build" "github.com/spf13/cobra" ) var ( stopCmd = &cobra.Command{ Use: "stop", Short: "Stop the Sia daemon", Long: "Stop the Sia daemon.", Run: wrap(stopcmd), } updateCmd = &cobra.Command{ Use: "update", Short: "Update Sia", Long: "Check for (and/or download) available updates for Sia.", Run: wrap(updatecmd), } updateCheckCmd = &cobra.Command{ Use: "check", Short: "Check for available updates", Long: "Check for available updates.", Run: wrap(updatecheckcmd), } versionCmd = &cobra.Command{ Use: "version", Short: "Print version information", Long: "Print version information.", Run: wrap(versioncmd), } ) type updateInfo struct { Available bool `json:"available"` Version string `json:"version"` } type daemonVersion struct { Version string } // version prints the version of siac and siad. func versioncmd() { fmt.Println("Sia Client v" + build.Version) var versioninfo daemonVersion err := getAPI("/daemon/version", &versioninfo) if err != nil { fmt.Println("Could not get daemon version:", err) return } fmt.Println("Sia Daemon v" + versioninfo.Version) } // stopcmd is the handler for the command `siac stop`. // Stops the daemon. func stopcmd() { err := get("/daemon/stop") if err != nil { die("Could not stop daemon:", err) } fmt.Println("Sia daemon stopped.") } func updatecmd() { var update updateInfo err := getAPI("/daemon/update", &update) if err != nil { fmt.Println("Could not check for update:", err) return } if !update.Available { fmt.Println("Already up to date.") return } err = post("/daemon/update", "") if err != nil { fmt.Println("Could not apply update:", err) return } fmt.Printf("Updated to version %s! Restart siad now.\n", update.Version) } func updatecheckcmd() { var update updateInfo err := getAPI("/daemon/update", &update) if err != nil { fmt.Println("Could not check for update:", err) return } if update.Available { fmt.Printf("A new release (v%s) is available! Run 'siac update' to install it.\n", update.Version) } else { fmt.Println("Up to date.") } } Sia-1.0.4/siac/gatewaycmd.go000066400000000000000000000056331302033070500156230ustar00rootroot00000000000000package main import ( "fmt" "os" "text/tabwriter" "github.com/spf13/cobra" "github.com/NebulousLabs/Sia/api" ) var ( gatewayCmd = &cobra.Command{ Use: "gateway", Short: "Perform gateway actions", Long: "View and manage the gateway's connected peers.", Run: wrap(gatewaycmd), } gatewayConnectCmd = &cobra.Command{ Use: "connect [address]", Short: "Connect to a peer", Long: "Connect to a peer and add it to the node list.", Run: wrap(gatewayconnectcmd), } gatewayDisconnectCmd = &cobra.Command{ Use: "disconnect [address]", Short: "Disconnect from a peer", Long: "Disconnect from a peer. Does not remove the peer from the node list.", Run: wrap(gatewaydisconnectcmd), } gatewayAddressCmd = &cobra.Command{ Use: "address", Short: "Print the gateway address", Long: "Print the network address of the gateway.", Run: wrap(gatewayaddresscmd), } gatewayListCmd = &cobra.Command{ Use: "list", Short: "View a list of peers", Long: "View the current peer list.", Run: wrap(gatewaylistcmd), } ) // gatewayconnectcmd is the handler for the command `siac gateway add [address]`. // Adds a new peer to the peer list. func gatewayconnectcmd(addr string) { err := post("/gateway/connect/"+addr, "") if err != nil { die("Could not add peer:", err) } fmt.Println("Added", addr, "to peer list.") } // gatewaydisconnectcmd is the handler for the command `siac gateway remove [address]`. // Removes a peer from the peer list. func gatewaydisconnectcmd(addr string) { err := post("/gateway/disconnect/"+addr, "") if err != nil { die("Could not remove peer:", err) } fmt.Println("Removed", addr, "from peer list.") } // gatewayaddresscmd is the handler for the command `siac gateway address`. // Prints the gateway's network address. func gatewayaddresscmd() { var info api.GatewayGET err := getAPI("/gateway", &info) if err != nil { die("Could not get gateway address:", err) } fmt.Println("Address:", info.NetAddress) } // gatewaycmd is the handler for the command `siac gateway`. // Prints the gateway's network address and number of peers. func gatewaycmd() { var info api.GatewayGET err := getAPI("/gateway", &info) if err != nil { die("Could not get gateway address:", err) } fmt.Println("Address:", info.NetAddress) fmt.Println("Active peers:", len(info.Peers)) } // gatewaylistcmd is the handler for the command `siac gateway list`. // Prints a list of all peers. func gatewaylistcmd() { var info api.GatewayGET err := getAPI("/gateway", &info) if err != nil { die("Could not get peer list:", err) } if len(info.Peers) == 0 { fmt.Println("No peers to show.") return } fmt.Println(len(info.Peers), "active peers:") w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) fmt.Fprintln(w, "Version\tOutbound\tAddress") for _, peer := range info.Peers { fmt.Fprintf(w, "%v\t%v\t%v\n", peer.Version, yesNo(!peer.Inbound), peer.NetAddress) } w.Flush() } Sia-1.0.4/siac/hostcmd.go000066400000000000000000000303271302033070500151350ustar00rootroot00000000000000package main import ( "fmt" "math/big" "os" "text/tabwriter" "github.com/NebulousLabs/Sia/api" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" "github.com/spf13/cobra" ) var ( hostCmd = &cobra.Command{ Use: "host", Short: "Perform host actions", Long: "View or modify host settings.", Run: wrap(hostcmd), } hostConfigCmd = &cobra.Command{ Use: "config [setting] [value]", Short: "Modify host settings", Long: `Modify host settings. Available settings: acceptingcontracts: boolean maxduration: blocks maxdownloadbatchsize: bytes maxrevisebatchsize: bytes netaddress: string windowsize: blocks collateral: currency collateralbudget: currency maxcollateral: currency mincontractprice: currency mindownloadbandwidthprice: currency / TB minstorageprice: currency / TB / Month minuploadbandwidthprice: currency / TB Currency units can be specified, e.g. 10SC; run 'siac help wallet' for details. Blocks are approximately 10 minutes each. For a description of each parameter, see doc/API.md. To configure the host to accept new contracts, set acceptingcontracts to true: siac host config acceptingcontracts true `, Run: wrap(hostconfigcmd), } hostAnnounceCmd = &cobra.Command{ Use: "announce", Short: "Announce yourself as a host", Long: `Announce yourself as a host on the network. Announcing will also configure the host to start accepting contracts. You can revert this by running: siac host config acceptingcontracts false You may also supply a specific address to be announced, e.g.: siac host announce my-host-domain.com:9001 Doing so will override the standard connectivity checks.`, Run: hostannouncecmd, } hostFolderCmd = &cobra.Command{ Use: "folder", Short: "Add, remove, or resize a storage folder", Long: "Add, remove, or resize a storage folder.", } hostFolderAddCmd = &cobra.Command{ Use: "add [path] [size]", Short: "Add a storage folder to the host", Long: "Add a storage folder to the host, specifying how much data it should store", Run: wrap(hostfolderaddcmd), } hostFolderRemoveCmd = &cobra.Command{ Use: "remove [path]", Short: "Remove a storage folder from the host", Long: `Remove a storage folder from the host. Note that this does not delete any data; it will instead be distributed across the remaining storage folders.`, Run: wrap(hostfolderremovecmd), } hostFolderResizeCmd = &cobra.Command{ Use: "resize [path] [size]", Short: "Resize a storage folder", Long: `Change how much data a storage folder should store. If the new size is less than what the folder is currently storing, data will be distributed across the other storage folders.`, Run: wrap(hostfolderresizecmd), } hostSectorCmd = &cobra.Command{ Use: "sector", Short: "Add or delete a sector (add not supported)", Long: `Add or delete a sector. Adding is not currently supported. Note that deleting a sector may impact host revenue.`, } hostSectorDeleteCmd = &cobra.Command{ Use: "delete [root]", Short: "Delete a sector", Long: `Delete a sector, identified by its Merkle root. Note that deleting a sector may impact host revenue.`, Run: wrap(hostsectordeletecmd), } ) // hostcmd is the handler for the command `siac host`. // Prints info about the host and its storage folders. func hostcmd() { hg := new(api.HostGET) err := getAPI("/host", hg) if err != nil { die("Could not fetch host settings:", err) } sg := new(api.StorageGET) err = getAPI("/host/storage", sg) if err != nil { die("Could not fetch storage info:", err) } // Determine the competitive price string. ah := new(api.ActiveHosts) var competitivePrice string err = getAPI("/hostdb/active?numhosts=24", ah) if err != nil || len(ah.Hosts) == 0 { competitivePrice = "Unavailable" } else { var sum types.Currency for _, host := range ah.Hosts { sum = sum.Add(host.StoragePrice) } // Divide by the number of hosts to get the average price, and then // trim 5% to present what would be a competitive edge. competitivePrice = currencyUnits(sum.Div64(uint64(len(ah.Hosts))).MulFloat(0.95).Mul(modules.BlockBytesPerMonthTerabyte)) } es := hg.ExternalSettings fm := hg.FinancialMetrics is := hg.InternalSettings nm := hg.NetworkMetrics // calculate total storage available and remaining var totalstorage, storageremaining uint64 for _, folder := range sg.Folders { totalstorage += folder.Capacity storageremaining += folder.CapacityRemaining } // convert price from bytes/block to TB/Month price := currencyUnits(is.MinStoragePrice.Mul(modules.BlockBytesPerMonthTerabyte)) // calculate total revenue totalRevenue := fm.ContractCompensation. Add(fm.StorageRevenue). Add(fm.DownloadBandwidthRevenue). Add(fm.UploadBandwidthRevenue) totalPotentialRevenue := fm.PotentialContractCompensation. Add(fm.PotentialStorageRevenue). Add(fm.PotentialDownloadBandwidthRevenue). Add(fm.PotentialUploadBandwidthRevenue) // determine the display method for the net address. netaddr := es.NetAddress if is.NetAddress == "" { netaddr += " (automatically determined)" } else { netaddr += " (manually specified)" } if hostVerbose { // describe net address fmt.Printf(`General Info: Estimated Competitive Price: %v Host Internal Settings: acceptingcontracts: %v maxduration: %v Weeks maxdownloadbatchsize: %v maxrevisebatchsize: %v netaddress: %v windowsize: %v Hours collateral: %v / TB / Month collateralbudget: %v maxcollateral: %v Per Contract mincontractprice: %v mindownloadbandwidthprice: %v / TB minstorageprice: %v / TB / Month minuploadbandwidthprice: %v / TB Host Financials: Contract Count: %v Transaction Fee Compensation: %v Potential Fee Compensation: %v Transaction Fee Expenses: %v Storage Revenue: %v Potential Storage Revenue: %v Locked Collateral: %v Risked Collateral: %v Lost Collateral: %v Download Revenue: %v Potential Download Revenue: %v Upload Revenue : %v Potential Upload Revenue: %v RPC Stats: Error Calls: %v Unrecognized Calls: %v Download Calls: %v Renew Calls: %v Revise Calls: %v Settings Calls: %v FormContract Calls: %v `, competitivePrice, yesNo(is.AcceptingContracts), periodUnits(is.MaxDuration), filesizeUnits(int64(is.MaxDownloadBatchSize)), filesizeUnits(int64(is.MaxReviseBatchSize)), netaddr, is.WindowSize/6, currencyUnits(is.Collateral.Mul(modules.BlockBytesPerMonthTerabyte)), currencyUnits(is.CollateralBudget), currencyUnits(is.MaxCollateral), currencyUnits(is.MinContractPrice), currencyUnits(is.MinDownloadBandwidthPrice.Mul(modules.BytesPerTerabyte)), currencyUnits(is.MinStoragePrice.Mul(modules.BlockBytesPerMonthTerabyte)), currencyUnits(is.MinUploadBandwidthPrice.Mul(modules.BytesPerTerabyte)), fm.ContractCount, currencyUnits(fm.ContractCompensation), currencyUnits(fm.PotentialContractCompensation), currencyUnits(fm.TransactionFeeExpenses), currencyUnits(fm.StorageRevenue), currencyUnits(fm.PotentialStorageRevenue), currencyUnits(fm.LockedStorageCollateral), currencyUnits(fm.RiskedStorageCollateral), currencyUnits(fm.LostStorageCollateral), currencyUnits(fm.DownloadBandwidthRevenue), currencyUnits(fm.PotentialDownloadBandwidthRevenue), currencyUnits(fm.UploadBandwidthRevenue), currencyUnits(fm.PotentialUploadBandwidthRevenue), nm.ErrorCalls, nm.UnrecognizedCalls, nm.DownloadCalls, nm.RenewCalls, nm.ReviseCalls, nm.SettingsCalls, nm.FormContractCalls) } else { fmt.Printf(`Host info: Estimated Competitive Price: %v Storage: %v (%v used) Price: %v / TB / Month Max Duration: %v Weeks Accepting Contracts: %v Anticipated Revenue: %v Locked Collateral: %v Revenue: %v `, competitivePrice, filesizeUnits(int64(totalstorage)), filesizeUnits(int64(totalstorage-storageremaining)), price, periodUnits(is.MaxDuration), yesNo(is.AcceptingContracts), currencyUnits(totalPotentialRevenue), currencyUnits(fm.LockedStorageCollateral), currencyUnits(totalRevenue)) } fmt.Println("\nStorage Folders:") // display storage folder info if len(sg.Folders) == 0 { fmt.Println("No storage folders configured") return } w := tabwriter.NewWriter(os.Stdout, 0, 0, 4, ' ', 0) fmt.Fprintf(w, "\tUsed\tCapacity\t%% Used\tPath\n") for _, folder := range sg.Folders { curSize := int64(folder.Capacity - folder.CapacityRemaining) pctUsed := 100 * (float64(curSize) / float64(folder.Capacity)) fmt.Fprintf(w, "\t%s\t%s\t%.2f\t%s\n", filesizeUnits(curSize), filesizeUnits(int64(folder.Capacity)), pctUsed, folder.Path) } w.Flush() } // hostconfigcmd is the handler for the command `siac host config [setting] [value]`. // Modifies host settings. func hostconfigcmd(param, value string) { switch param { // currency (convert to hastings) case "collateralbudget", "maxcollateral", "mincontractprice": hastings, err := parseCurrency(value) if err != nil { die("Could not parse "+param+":", err) } value = hastings // currency/TB (convert to hastings/byte) case "mindownloadbandwidthprice", "minuploadbandwidthprice": hastings, err := parseCurrency(value) if err != nil { die("Could not parse "+param+":", err) } i, _ := new(big.Int).SetString(hastings, 10) c := types.NewCurrency(i).Div(modules.BytesPerTerabyte) value = c.String() // currency/TB/month (convert to hastings/byte/block) case "collateral", "minstorageprice": hastings, err := parseCurrency(value) if err != nil { die("Could not parse "+param+":", err) } i, _ := new(big.Int).SetString(hastings, 10) c := types.NewCurrency(i).Div(modules.BlockBytesPerMonthTerabyte) value = c.String() // other valid settings case "acceptingcontracts", "maxdownloadbatchsize", "maxduration", "maxrevisebatchsize", "netaddress", "windowsize": // invalid settings default: die("\"" + param + "\" is not a host setting") } err := post("/host", param+"="+value) if err != nil { die("Could not update host settings:", err) } fmt.Println("Host settings updated.") } // hostannouncecmd is the handler for the command `siac host announce`. // Announces yourself as a host to the network. Optionally takes an address to // announce as. func hostannouncecmd(cmd *cobra.Command, args []string) { var err error switch len(args) { case 0: err = post("/host/announce", "") case 1: err = post("/host/announce", "netaddress="+args[0]) default: cmd.UsageFunc()(cmd) os.Exit(exitCodeUsage) } if err != nil { die("Could not announce host:", err) } fmt.Println("Host announcement submitted to network.") // start accepting contracts err = post("/host", "acceptingcontracts=true") if err != nil { die("Could not configure host to accept contracts:", err) } fmt.Println(` The host has also been configured to accept contracts. To revert this, run: siac host config acceptingcontracts false `) } // hostfolderaddcmd adds a folder to the host. func hostfolderaddcmd(path, size string) { size, err := parseFilesize(size) if err != nil { die("Could not parse size:", err) } err = post("/host/storage/folders/add", fmt.Sprintf("path=%s&size=%s", abs(path), size)) if err != nil { die("Could not add folder:", err) } fmt.Println("Added folder", path) } // hostfolderremovecmd removes a folder from the host. func hostfolderremovecmd(path string) { err := post("/host/storage/folders/remove", "path="+abs(path)) if err != nil { die("Could not remove folder:", err) } fmt.Println("Removed folder", path) } // hostfolderresizecmd resizes a folder in the host. func hostfolderresizecmd(path, newsize string) { newsize, err := parseFilesize(newsize) if err != nil { die("Could not parse size:", err) } err = post("/host/storage/folders/resize", fmt.Sprintf("path=%s&newsize=%s", abs(path), newsize)) if err != nil { die("Could not resize folder:", err) } fmt.Printf("Resized folder %v to %v\n", path, newsize) } // hostsectordeletecmd deletes a sector from the host. func hostsectordeletecmd(root string) { err := post("/host/storage/sectors/delete/"+root, "") if err != nil { die("Could not delete sector:", err) } fmt.Println("Deleted sector", root) } Sia-1.0.4/siac/hostdbcmd.go000066400000000000000000000014171302033070500154410ustar00rootroot00000000000000package main import ( "fmt" "github.com/spf13/cobra" "github.com/NebulousLabs/Sia/api" "github.com/NebulousLabs/Sia/modules" ) var ( hostdbCmd = &cobra.Command{ Use: "hostdb", Short: "View or modify the host database", Long: "Add and remove hosts, or list active hosts on the network.", Run: wrap(hostdbcmd), } ) func hostdbcmd() { info := new(api.ActiveHosts) err := getAPI("/hostdb/active", info) if err != nil { die("Could not fetch host list:", err) } if len(info.Hosts) == 0 { fmt.Println("No known active hosts") return } fmt.Println("Active hosts:") for _, host := range info.Hosts { price := host.StoragePrice.Mul(modules.BlockBytesPerMonthTerabyte) fmt.Printf("\t%v - %v / TB / Month\n", host.NetAddress, currencyUnits(price)) } } Sia-1.0.4/siac/main.go000066400000000000000000000207031302033070500144150ustar00rootroot00000000000000package main import ( "encoding/json" "errors" "fmt" "net" "net/http" "os" "reflect" "github.com/bgentry/speakeasy" "github.com/spf13/cobra" "github.com/NebulousLabs/Sia/api" "github.com/NebulousLabs/Sia/build" ) // flags var ( addr string // override default API address initPassword bool // supply a custom password when creating a wallet hostVerbose bool // display additional host info renterShowHistory bool // Show download history in addition to download queue. renterListVerbose bool // Show additional info about uploaded files. ) // exit codes // inspired by sysexits.h const ( exitCodeGeneral = 1 // Not in sysexits.h, but is standard practice. exitCodeUsage = 64 // EX_USAGE in sysexits.h ) // non2xx returns true for non-success HTTP status codes. func non2xx(code int) bool { return code < 200 || code > 299 } // decodeError returns the api.Error from a API response. This method should // only be called if the response's status code is non-2xx. The error returned // may not be of type api.Error in the event of an error unmarshalling the // JSON. func decodeError(resp *http.Response) error { var apiErr api.Error err := json.NewDecoder(resp.Body).Decode(&apiErr) if err != nil { return err } return apiErr } // apiGet wraps a GET request with a status code check, such that if the GET does // not return 2xx, the error will be read and returned. The response body is // not closed. func apiGet(call string) (*http.Response, error) { if host, port, _ := net.SplitHostPort(addr); host == "" { addr = net.JoinHostPort("localhost", port) } resp, err := api.HttpGET("http://" + addr + call) if err != nil { return nil, errors.New("no response from daemon") } // check error code if resp.StatusCode == http.StatusUnauthorized { resp.Body.Close() // Prompt for password and retry request with authentication. password, err := speakeasy.Ask("API password: ") if err != nil { return nil, err } resp, err = api.HttpGETAuthenticated("http://"+addr+call, password) if err != nil { return nil, errors.New("no response from daemon - authentication failed") } } if resp.StatusCode == http.StatusNotFound { resp.Body.Close() return nil, errors.New("API call not recognized: " + call) } if non2xx(resp.StatusCode) { err := decodeError(resp) resp.Body.Close() return nil, err } return resp, nil } // getAPI makes a GET API call and decodes the response. An error is returned // if the response status is not 2xx. func getAPI(call string, obj interface{}) error { resp, err := apiGet(call) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode == http.StatusNoContent { return errors.New("expecting a response, but API returned status code 204 No Content") } err = json.NewDecoder(resp.Body).Decode(obj) if err != nil { return err } return nil } // get makes an API call and discards the response. An error is returned if the // response status is not 2xx. func get(call string) error { resp, err := apiGet(call) if err != nil { return err } resp.Body.Close() return nil } // apiPost wraps a POST request with a status code check, such that if the POST // does not return 2xx, the error will be read and returned. The response body // is not closed. func apiPost(call, vals string) (*http.Response, error) { if host, port, _ := net.SplitHostPort(addr); host == "" { addr = net.JoinHostPort("localhost", port) } resp, err := api.HttpPOST("http://"+addr+call, vals) if err != nil { return nil, errors.New("no response from daemon") } // check error code if resp.StatusCode == http.StatusUnauthorized { resp.Body.Close() // Prompt for password and retry request with authentication. password, err := speakeasy.Ask("API password: ") if err != nil { return nil, err } resp, err = api.HttpPOSTAuthenticated("http://"+addr+call, vals, password) if err != nil { return nil, errors.New("no response from daemon - authentication failed") } } if resp.StatusCode == http.StatusNotFound { resp.Body.Close() return nil, errors.New("API call not recognized: " + call) } if non2xx(resp.StatusCode) { err := decodeError(resp) resp.Body.Close() return nil, err } return resp, nil } // postResp makes a POST API call and decodes the response. An error is // returned if the response status is not 2xx. func postResp(call, vals string, obj interface{}) error { resp, err := apiPost(call, vals) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode == http.StatusNoContent { return errors.New("expecting a response, but API returned status code 204 No Content") } err = json.NewDecoder(resp.Body).Decode(obj) if err != nil { return err } return nil } // post makes an API call and discards the response. An error is returned if // the response status is not 2xx. func post(call, vals string) error { resp, err := apiPost(call, vals) if err != nil { return err } resp.Body.Close() return nil } // wrap wraps a generic command with a check that the command has been // passed the correct number of arguments. The command must take only strings // as arguments. func wrap(fn interface{}) func(*cobra.Command, []string) { fnVal, fnType := reflect.ValueOf(fn), reflect.TypeOf(fn) if fnType.Kind() != reflect.Func { panic("wrapped function has wrong type signature") } for i := 0; i < fnType.NumIn(); i++ { if fnType.In(i).Kind() != reflect.String { panic("wrapped function has wrong type signature") } } return func(cmd *cobra.Command, args []string) { if len(args) != fnType.NumIn() { cmd.UsageFunc()(cmd) os.Exit(exitCodeUsage) } argVals := make([]reflect.Value, fnType.NumIn()) for i := range args { argVals[i] = reflect.ValueOf(args[i]) } fnVal.Call(argVals) } } // die prints its arguments to stderr, then exits the program with the default // error code. func die(args ...interface{}) { fmt.Fprintln(os.Stderr, args...) os.Exit(exitCodeGeneral) } func main() { root := &cobra.Command{ Use: os.Args[0], Short: "Sia Client v" + build.Version, Long: "Sia Client v" + build.Version, Run: wrap(consensuscmd), } // create command tree root.AddCommand(versionCmd) root.AddCommand(stopCmd) root.AddCommand(updateCmd) updateCmd.AddCommand(updateCheckCmd) root.AddCommand(hostCmd) hostCmd.AddCommand(hostConfigCmd, hostAnnounceCmd, hostFolderCmd, hostSectorCmd) hostFolderCmd.AddCommand(hostFolderAddCmd, hostFolderRemoveCmd, hostFolderResizeCmd) hostSectorCmd.AddCommand(hostSectorDeleteCmd) hostCmd.Flags().BoolVarP(&hostVerbose, "verbose", "v", false, "Display detailed host info") root.AddCommand(hostdbCmd) root.AddCommand(minerCmd) minerCmd.AddCommand(minerStartCmd, minerStopCmd) root.AddCommand(walletCmd) walletCmd.AddCommand(walletAddressCmd, walletAddressesCmd, walletInitCmd, walletLoadCmd, walletLockCmd, walletSeedsCmd, walletSendCmd, walletBalanceCmd, walletTransactionsCmd, walletUnlockCmd) walletInitCmd.Flags().BoolVarP(&initPassword, "password", "p", false, "Prompt for a custom password") walletLoadCmd.AddCommand(walletLoad033xCmd, walletLoadSeedCmd, walletLoadSiagCmd) walletSendCmd.AddCommand(walletSendSiacoinsCmd, walletSendSiafundsCmd) root.AddCommand(renterCmd) renterCmd.AddCommand(renterFilesDeleteCmd, renterFilesDownloadCmd, renterDownloadsCmd, renterAllowanceCmd, renterSetAllowanceCmd, renterContractsCmd, renterFilesListCmd, renterFilesRenameCmd, renterFilesUploadCmd, renterUploadsCmd) renterCmd.Flags().BoolVarP(&renterListVerbose, "verbose", "v", false, "Show additional file info such as redundancy") renterDownloadsCmd.Flags().BoolVarP(&renterShowHistory, "history", "H", false, "Show download history in addition to the download queue") renterFilesListCmd.Flags().BoolVarP(&renterListVerbose, "verbose", "v", false, "Show additional file info such as redundancy") root.AddCommand(gatewayCmd) gatewayCmd.AddCommand(gatewayConnectCmd, gatewayDisconnectCmd, gatewayAddressCmd, gatewayListCmd) root.AddCommand(consensusCmd) // parse flags root.PersistentFlags().StringVarP(&addr, "addr", "a", "localhost:9980", "which host/port to communicate with (i.e. the host/port siad is listening on)") // run if err := root.Execute(); err != nil { // Since no commands return errors (all commands set Command.Run instead of // Command.RunE), Command.Execute() should only return an error on an // invalid command or flag. Therefore Command.Usage() was called (assuming // Command.SilenceUsage is false) and we should exit with exitCodeUsage. os.Exit(exitCodeUsage) } } Sia-1.0.4/siac/minercmd.go000066400000000000000000000030661302033070500152720ustar00rootroot00000000000000package main import ( "fmt" "github.com/NebulousLabs/Sia/api" "github.com/spf13/cobra" ) var ( minerCmd = &cobra.Command{ Use: "miner", Short: "Perform miner actions", Long: "Perform miner actions and view miner status.", Run: wrap(minercmd), } minerStartCmd = &cobra.Command{ Use: "start", Short: "Start cpu mining", Long: "Start cpu mining, if the miner is already running, this command does nothing", Run: wrap(minerstartcmd), } minerStopCmd = &cobra.Command{ Use: "stop", Short: "Stop mining", Long: "Stop mining (this may take a few moments).", Run: wrap(minerstopcmd), } ) // minerstartcmd is the handler for the command `siac miner start`. // Starts the CPU miner. func minerstartcmd() { err := get("/miner/start") if err != nil { die("Could not start miner:", err) } fmt.Println("CPU Miner is now running.") } // minercmd is the handler for the command `siac miner`. // Prints the status of the miner. func minercmd() { status := new(api.MinerGET) err := getAPI("/miner", status) if err != nil { die("Could not get miner status:", err) } miningStr := "off" if status.CPUMining { miningStr = "on" } fmt.Printf(`Miner status: CPU Mining: %s CPU Hashrate: %v KH/s Blocks Mined: %d (%d stale) `, miningStr, status.CPUHashrate/1000, status.BlocksMined, status.StaleBlocksMined) } // minerstopcmd is the handler for the command `siac miner stop`. // Stops the CPU miner. func minerstopcmd() { err := get("/miner/stop") if err != nil { die("Could not stop miner:", err) } fmt.Println("Stopped mining.") } Sia-1.0.4/siac/parse.go000066400000000000000000000075721302033070500146140ustar00rootroot00000000000000package main import ( "errors" "fmt" "math" "math/big" "strings" "github.com/NebulousLabs/Sia/types" ) var errUnableToParseSize = errors.New("unable to parse size") // filesize returns a string that displays a filesize in human-readable units. func filesizeUnits(size int64) string { if size == 0 { return "0 B" } sizes := []string{"B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"} i := int(math.Log10(float64(size)) / 3) return fmt.Sprintf("%.*f %s", i, float64(size)/math.Pow10(3*i), sizes[i]) } // parseFilesize converts strings of form 10GB to a size in bytes. Fractional // sizes are truncated at the byte size. func parseFilesize(strSize string) (string, error) { units := []struct { suffix string multiplier int64 }{ {"kb", 1e3}, {"mb", 1e6}, {"gb", 1e9}, {"tb", 1e12}, {"kib", 1 << 10}, {"mib", 1 << 20}, {"gib", 1 << 30}, {"tib", 1 << 40}, {"b", 1}, // must be after others else it'll match on them all {"", 1}, // no suffix is still a valid suffix } strSize = strings.ToLower(strSize) for _, unit := range units { if strings.HasSuffix(strSize, unit.suffix) { r, ok := new(big.Rat).SetString(strings.TrimSuffix(strSize, unit.suffix)) if !ok { return "", errUnableToParseSize } r.Mul(r, new(big.Rat).SetInt(big.NewInt(unit.multiplier))) if !r.IsInt() { f, _ := r.Float64() return fmt.Sprintf("%d", int64(f)), nil } return r.RatString(), nil } } return "", errUnableToParseSize } // periodUnits turns a period in terms of blocks to a number of weeks. func periodUnits(blocks types.BlockHeight) string { return fmt.Sprint(blocks / 1008) // 1008 blocks per week } // parsePeriod converts a number of weeks to a number of blocks. func parsePeriod(period string) (string, error) { var weeks float64 _, err := fmt.Sscan(period, &weeks) if err != nil { return "", errUnableToParseSize } blocks := int(weeks * 1008) // 1008 blocks per week return fmt.Sprint(blocks), nil } // currencyUnits converts a types.Currency to a string with human-readable // units. The unit used will be the largest unit that results in a value // greater than 1. The value is rounded to 4 significant digits. func currencyUnits(c types.Currency) string { pico := types.SiacoinPrecision.Div64(1e12) if c.Cmp(pico) < 0 { return c.String() + " H" } // iterate until we find a unit greater than c mag := pico unit := "" for _, unit = range []string{"pS", "nS", "uS", "mS", "SC", "KS", "MS", "GS", "TS"} { if c.Cmp(mag.Mul64(1e3)) < 0 { break } else if unit != "TS" { // don't want to perform this multiply on the last iter; that // would give us 1.235 TS instead of 1235 TS mag = mag.Mul64(1e3) } } num := new(big.Rat).SetInt(c.Big()) denom := new(big.Rat).SetInt(mag.Big()) res, _ := new(big.Rat).Mul(num, denom.Inv(denom)).Float64() return fmt.Sprintf("%.4g %s", res, unit) } // parseCurrency converts a siacoin amount to base units. func parseCurrency(amount string) (string, error) { units := []string{"pS", "nS", "uS", "mS", "SC", "KS", "MS", "GS", "TS"} for i, unit := range units { if strings.HasSuffix(amount, unit) { // scan into big.Rat r, ok := new(big.Rat).SetString(strings.TrimSuffix(amount, unit)) if !ok { return "", errors.New("malformed amount") } // convert units exp := 24 + 3*(int64(i)-4) mag := new(big.Int).Exp(big.NewInt(10), big.NewInt(exp), nil) r.Mul(r, new(big.Rat).SetInt(mag)) // r must be an integer at this point if !r.IsInt() { return "", errors.New("non-integer number of hastings") } return r.RatString(), nil } } // check for hastings separately if strings.HasSuffix(amount, "H") { return strings.TrimSuffix(amount, "H"), nil } return "", errors.New("amount is missing units; run 'wallet --help' for a list of units") } // yesNo returns "Yes" if b is true, and "No" if b is false. func yesNo(b bool) string { if b { return "Yes" } return "No" } Sia-1.0.4/siac/parse_test.go000066400000000000000000000037731302033070500156520ustar00rootroot00000000000000package main import ( "math/big" "testing" "github.com/NebulousLabs/Sia/types" ) func TestParseFilesize(t *testing.T) { tests := []struct { in, out string err error }{ {"1b", "1", nil}, {"1KB", "1000", nil}, {"1MB", "1000000", nil}, {"1GB", "1000000000", nil}, {"1TB", "1000000000000", nil}, {"1KiB", "1024", nil}, {"1MiB", "1048576", nil}, {"1GiB", "1073741824", nil}, {"1TiB", "1099511627776", nil}, {"", "", errUnableToParseSize}, {"123", "123", nil}, {"123TB", "123000000000000", nil}, {"123GiB", "132070244352", nil}, {"123BiB", "", errUnableToParseSize}, {"GB", "", errUnableToParseSize}, {"123G", "", errUnableToParseSize}, {"123B99", "", errUnableToParseSize}, {"12A3456", "", errUnableToParseSize}, {"1.23KB", "1230", nil}, {"1.234KB", "1234", nil}, {"1.2345KB", "1234", nil}, } for _, test := range tests { res, err := parseFilesize(test.in) if res != test.out || err != test.err { t.Errorf("parseFilesize(%v): expected %v %v, got %v %v", test.in, test.out, test.err, res, err) } } } func TestCurrencyUnits(t *testing.T) { tests := []struct { in, out string }{ {"1", "1 H"}, {"1000", "1000 H"}, {"100000000000", "100000000000 H"}, {"1000000000000", "1 pS"}, {"1234560000000", "1.235 pS"}, {"12345600000000", "12.35 pS"}, {"123456000000000", "123.5 pS"}, {"1000000000000000", "1 nS"}, {"1000000000000000000", "1 uS"}, {"1000000000000000000000", "1 mS"}, {"1000000000000000000000000", "1 SC"}, {"1000000000000000000000000000", "1 KS"}, {"1000000000000000000000000000000", "1 MS"}, {"1000000000000000000000000000000000", "1 GS"}, {"1000000000000000000000000000000000000", "1 TS"}, {"1234560000000000000000000000000000000", "1.235 TS"}, {"1234560000000000000000000000000000000000", "1235 TS"}, } for _, test := range tests { i, _ := new(big.Int).SetString(test.in, 10) out := currencyUnits(types.NewCurrency(i)) if out != test.out { t.Errorf("currencyUnits(%v): expected %v, got %v", test.in, test.out, out) } } } Sia-1.0.4/siac/rentercmd.go000066400000000000000000000264431302033070500154630ustar00rootroot00000000000000package main import ( "fmt" "os" "path/filepath" "sort" "text/tabwriter" "github.com/spf13/cobra" "github.com/NebulousLabs/Sia/api" "github.com/NebulousLabs/Sia/modules" ) var ( renterCmd = &cobra.Command{ Use: "renter", Short: "Perform renter actions", Long: "Upload, download, rename, delete, load, or share files.", Run: wrap(rentercmd), } renterUploadsCmd = &cobra.Command{ Use: "uploads", Short: "View the upload queue", Long: "View the list of files currently uploading.", Run: wrap(renteruploadscmd), } renterDownloadsCmd = &cobra.Command{ Use: "downloads", Short: "View the download queue", Long: "View the list of files currently downloading.", Run: wrap(renterdownloadscmd), } renterAllowanceCmd = &cobra.Command{ Use: "allowance", Short: "View the current allowance", Long: "View the current allowance, which controls how much money is spent on file contracts.", Run: wrap(renterallowancecmd), } renterSetAllowanceCmd = &cobra.Command{ Use: "setallowance [amount] [period]", Short: "Set the allowance", Long: `Set the amount of money that can be spent over a given period. amount is given in currency units (SC, KS, etc.) period is given in weeks; 1 week is roughly 1000 blocks Note that setting the allowance will cause siad to immediately begin forming contracts! You should only set the allowance once you are fully synced and you have a reasonable number (>30) of hosts in your hostdb.`, Run: wrap(rentersetallowancecmd), } renterContractsCmd = &cobra.Command{ Use: "contracts", Short: "View the Renter's contracts", Long: "View the contracts that the Renter has formed with hosts.", Run: wrap(rentercontractscmd), } renterFilesDeleteCmd = &cobra.Command{ Use: "delete [path]", Aliases: []string{"rm"}, Short: "Delete a file", Long: "Delete a file. Does not delete the file on disk.", Run: wrap(renterfilesdeletecmd), } renterFilesDownloadCmd = &cobra.Command{ Use: "download [path] [destination]", Short: "Download a file", Long: "Download a previously-uploaded file to a specified destination.", Run: wrap(renterfilesdownloadcmd), } renterFilesListCmd = &cobra.Command{ Use: "list", Aliases: []string{"ls"}, Short: "List the status of all files", Long: "List the status of all files known to the renter on the Sia network.", Run: wrap(renterfileslistcmd), } renterFilesRenameCmd = &cobra.Command{ Use: "rename [path] [newpath]", Aliases: []string{"mv"}, Short: "Rename a file", Long: "Rename a file.", Run: wrap(renterfilesrenamecmd), } renterFilesUploadCmd = &cobra.Command{ Use: "upload [source] [path]", Short: "Upload a file", Long: "Upload a file to [path] on the Sia network.", Run: wrap(renterfilesuploadcmd), } ) // abs returns the absolute representation of a path. // TODO: bad things can happen if you run siac from a non-existent directory. // Implement some checks to catch this problem. func abs(path string) string { abspath, err := filepath.Abs(path) if err != nil { return path } return abspath } // rentercmd displays the renter's financial metrics and lists the files it is // tracking. func rentercmd() { var rg api.RenterGET err := getAPI("/renter", &rg) if err != nil { die("Could not get renter info:", err) } fm := rg.FinancialMetrics unspent := fm.ContractSpending.Sub(fm.DownloadSpending).Sub(fm.StorageSpending).Sub(fm.UploadSpending) fmt.Printf(`Renter info: Storage Spending: %v Upload Spending: %v Download Spending: %v Unspent Funds: %v Total Allocated: %v `, currencyUnits(fm.StorageSpending), currencyUnits(fm.UploadSpending), currencyUnits(fm.DownloadSpending), currencyUnits(unspent), currencyUnits(fm.ContractSpending)) // also list files renterfileslistcmd() } // renteruploadscmd is the handler for the command `siac renter uploads`. // Lists files currently uploading. func renteruploadscmd() { var rf api.RenterFiles err := getAPI("/renter/files", &rf) if err != nil { die("Could not get upload queue:", err) } // TODO: add a --history flag to the uploads command to mirror the --history // flag in the downloads command. This hasn't been done yet because the // call to /renter/files includes files that have been shared with you, // not just files you've uploaded. // Filter out files that have been uploaded. var filteredFiles []modules.FileInfo for _, fi := range rf.Files { if !fi.Available { filteredFiles = append(filteredFiles, fi) } } if len(filteredFiles) == 0 { fmt.Println("No files are uploading.") return } fmt.Println("Uploading", len(filteredFiles), "files:") for _, file := range filteredFiles { fmt.Printf("%13s %s (uploading, %0.2f%%)\n", filesizeUnits(int64(file.Filesize)), file.SiaPath, file.UploadProgress) } } // renterdownloadscmd is the handler for the command `siac renter downloads`. // Lists files currently downloading, and optionally previously downloaded // files if the -H or --history flag is specified. func renterdownloadscmd() { var queue api.RenterDownloadQueue err := getAPI("/renter/downloads", &queue) if err != nil { die("Could not get download queue:", err) } // Filter out files that have been downloaded. var downloading []modules.DownloadInfo for _, file := range queue.Downloads { if file.Received != file.Filesize { downloading = append(downloading, file) } } if len(downloading) == 0 { fmt.Println("No files are downloading.") } else { fmt.Println("Downloading", len(downloading), "files:") for _, file := range downloading { fmt.Printf("%s: %5.1f%% %s -> %s\n", file.StartTime.Format("Jan 02 03:04 PM"), 100*float64(file.Received)/float64(file.Filesize), file.SiaPath, file.Destination) } } if !renterShowHistory { return } fmt.Println() // Filter out files that are downloading. var downloaded []modules.DownloadInfo for _, file := range queue.Downloads { if file.Received == file.Filesize { downloaded = append(downloaded, file) } } if len(downloaded) == 0 { fmt.Println("No files downloaded.") } else { fmt.Println("Downloaded", len(downloaded), "files:") for _, file := range downloaded { fmt.Printf("%s: %s -> %s\n", file.StartTime.Format("Jan 02 03:04 PM"), file.SiaPath, file.Destination) } } } // renterallowancecmd displays the current allowance. func renterallowancecmd() { var rg api.RenterGET err := getAPI("/renter", &rg) if err != nil { die("Could not get allowance:", err) } allowance := rg.Settings.Allowance // convert to SC fmt.Printf(`Allowance: Amount: %v Period: %v blocks `, currencyUnits(allowance.Funds), allowance.Period) } // rentersetallowancecmd allows the user to set the allowance. func rentersetallowancecmd(amount, period string) { hastings, err := parseCurrency(amount) if err != nil { die("Could not parse amount:", err) } blocks, err := parsePeriod(period) if err != nil { die("Could not parse period") } err = post("/renter", fmt.Sprintf("funds=%s&period=%s", hastings, blocks)) if err != nil { die("Could not set allowance:", err) } fmt.Println("Allowance updated.") } // byValue sorts contracts by their value in siacoins, high to low. If two // contracts have the same value, they are sorted by their host's address. type byValue []api.RenterContract func (s byValue) Len() int { return len(s) } func (s byValue) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s byValue) Less(i, j int) bool { cmp := s[i].RenterFunds.Cmp(s[j].RenterFunds) if cmp == 0 { return s[i].NetAddress < s[j].NetAddress } return cmp > 0 } // rentercontractscmd is the handler for the comand `siac renter contracts`. // It lists the Renter's contracts. func rentercontractscmd() { var rc api.RenterContracts err := getAPI("/renter/contracts", &rc) if err != nil { die("Could not get contracts:", err) } if len(rc.Contracts) == 0 { fmt.Println("No contracts have been formed.") return } sort.Sort(byValue(rc.Contracts)) fmt.Println("Contracts:") w := tabwriter.NewWriter(os.Stdout, 2, 0, 2, ' ', 0) fmt.Fprintln(w, "Host\tValue\tData\tEnd Height\tID") for _, c := range rc.Contracts { fmt.Fprintf(w, "%v\t%8s\t%v\t%v\t%v\n", c.NetAddress, currencyUnits(c.RenterFunds), filesizeUnits(int64(c.Size)), c.EndHeight, c.ID) } w.Flush() } // renterfilesdeletecmd is the handler for the command `siac renter delete [path]`. // Removes the specified path from the Sia network. func renterfilesdeletecmd(path string) { err := post("/renter/delete/"+path, "") if err != nil { die("Could not delete file:", err) } fmt.Println("Deleted", path) } // renterfilesdownloadcmd is the handler for the comand `siac renter download [path] [destination]`. // Downloads a path from the Sia network to the local specified destination. func renterfilesdownloadcmd(path, destination string) { err := get("/renter/download/" + path + "?destination=" + abs(destination)) if err != nil { die("Could not download file:", err) } fmt.Printf("Downloaded '%s' to %s.\n", path, abs(destination)) } // bySiaPath implements sort.Interface for [] modules.FileInfo based on the // SiaPath field. type bySiaPath []modules.FileInfo func (s bySiaPath) Len() int { return len(s) } func (s bySiaPath) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s bySiaPath) Less(i, j int) bool { return s[i].SiaPath < s[j].SiaPath } // renterfileslistcmd is the handler for the command `siac renter list`. // Lists files known to the renter on the network. func renterfileslistcmd() { var rf api.RenterFiles err := getAPI("/renter/files", &rf) if err != nil { die("Could not get file list:", err) } if len(rf.Files) == 0 { fmt.Println("No files have been uploaded.") return } fmt.Println("Tracking", len(rf.Files), "files:") w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) if renterListVerbose { fmt.Fprintln(w, "File size\tAvailable\tProgress\tRedundancy\tRenewing\tSia path") } sort.Sort(bySiaPath(rf.Files)) for _, file := range rf.Files { fmt.Fprintf(w, "%9s", filesizeUnits(int64(file.Filesize))) if renterListVerbose { availableStr := yesNo(file.Available) renewingStr := yesNo(file.Renewing) redundancyStr := fmt.Sprintf("%.2f", file.Redundancy) if file.Redundancy == -1 { redundancyStr = "-" } uploadProgressStr := fmt.Sprintf("%.2f%%", file.UploadProgress) if file.UploadProgress == -1 { uploadProgressStr = "-" } fmt.Fprintf(w, "\t%s\t%8s\t%10s\t%s", availableStr, uploadProgressStr, redundancyStr, renewingStr) } fmt.Fprintf(w, "\t%s", file.SiaPath) if !renterListVerbose && !file.Available { fmt.Fprintf(w, " (uploading, %0.2f%%)", file.UploadProgress) } fmt.Fprintln(w, "") } w.Flush() } // renterfilesrenamecmd is the handler for the command `siac renter rename [path] [newpath]`. // Renames a file on the Sia network. func renterfilesrenamecmd(path, newpath string) { err := post("/renter/rename/"+path, "newsiapath="+newpath) if err != nil { die("Could not rename file:", err) } fmt.Printf("Renamed %s to %s\n", path, newpath) } // renterfilesuploadcmd is the handler for the command `siac renter upload [source] [path]`. // Uploads the [source] file to [path] on the Sia network. func renterfilesuploadcmd(source, path string) { err := post("/renter/upload/"+path, "source="+abs(source)) if err != nil { die("Could not upload file:", err) } fmt.Printf("Uploaded '%s' as %s.\n", abs(source), path) } Sia-1.0.4/siac/sort_test.go000066400000000000000000000020771302033070500155230ustar00rootroot00000000000000package main import ( "sort" "testing" "github.com/NebulousLabs/Sia/api" "github.com/NebulousLabs/Sia/types" ) // TestSortByValue tests that byValue sorts contracts correctly. func TestSortByValue(t *testing.T) { contracts := []api.RenterContract{ {RenterFunds: types.NewCurrency64(1), NetAddress: "b"}, {RenterFunds: types.NewCurrency64(4), NetAddress: "a"}, {RenterFunds: types.NewCurrency64(2), NetAddress: "c"}, {RenterFunds: types.NewCurrency64(5), NetAddress: "z"}, {RenterFunds: types.NewCurrency64(2), NetAddress: "c"}, {RenterFunds: types.NewCurrency64(0), NetAddress: "e"}, {RenterFunds: types.NewCurrency64(2), NetAddress: "a"}, } sort.Sort(byValue(contracts)) // check ordering for i := 0; i < len(contracts)-1; i++ { a, b := contracts[i], contracts[i+1] if cmp := a.RenterFunds.Cmp(b.RenterFunds); cmp < 0 { t.Error("contracts not primarily sorted by value:", a.RenterFunds, b.RenterFunds) } else if cmp == 0 && a.NetAddress > b.NetAddress { t.Error("contracts not secondarily sorted by address:", a.NetAddress, b.NetAddress) } } } Sia-1.0.4/siac/walletcmd.go000066400000000000000000000304061302033070500154460ustar00rootroot00000000000000package main import ( "fmt" "math/big" "github.com/bgentry/speakeasy" "github.com/spf13/cobra" "github.com/NebulousLabs/Sia/api" "github.com/NebulousLabs/Sia/types" ) var ( walletCmd = &cobra.Command{ Use: "wallet", Short: "Perform wallet actions", Long: `Generate a new address, send coins to another wallet, or view info about the wallet. Units: The smallest unit of siacoins is the hasting. One siacoin is 10^24 hastings. Other supported units are: pS (pico, 10^-12 SC) nS (nano, 10^-9 SC) uS (micro, 10^-6 SC) mS (milli, 10^-3 SC) SC KS (kilo, 10^3 SC) MS (mega, 10^6 SC) GS (giga, 10^9 SC) TS (tera, 10^12 SC)`, Run: wrap(walletbalancecmd), } walletAddressCmd = &cobra.Command{ Use: "address", Short: "Get a new wallet address", Long: "Generate a new wallet address from the wallet's primary seed.", Run: wrap(walletaddresscmd), } walletAddressesCmd = &cobra.Command{ Use: "addresses", Short: "List all addresses", Long: "List all addresses that have been generated by the wallet", Run: wrap(walletaddressescmd), } walletInitCmd = &cobra.Command{ Use: "init", Short: "Initialize and encrypt a new wallet", Long: `Generate a new wallet from a randomly generated seed, and encrypt it. By default the wallet encryption / unlock password is the same as the generated seed.`, Run: wrap(walletinitcmd), } walletLoadCmd = &cobra.Command{ Use: "load", Short: "Load a wallet seed, v0.3.3.x wallet, or siag keyset", // Run field is not set, as the load command itself is not a valid command. // A subcommand must be provided. } walletLoad033xCmd = &cobra.Command{ Use: "033x [filepath]", Short: "Load a v0.3.3.x wallet", Long: "Load a v0.3.3.x wallet into the current wallet", Run: wrap(walletload033xcmd), } walletLoadSeedCmd = &cobra.Command{ Use: `seed`, Short: "Add a seed to the wallet", Long: "Uses the given password to create a new wallet with that as the primary seed", Run: wrap(walletloadseedcmd), } walletLoadSiagCmd = &cobra.Command{ Use: `siag [filepath,...]`, Short: "Load siag key(s) into the wallet", Long: "Load siag key(s) into the wallet - typically used for siafunds.", Example: "siac wallet load siag key1.siakey,key2.siakey", Run: wrap(walletloadsiagcmd), } walletLockCmd = &cobra.Command{ Use: "lock", Short: "Lock the wallet", Long: "Lock the wallet, preventing further use", Run: wrap(walletlockcmd), } walletSeedsCmd = &cobra.Command{ Use: "seeds", Short: "Retrieve information about your seeds", Long: "Retrieves the current seed, how many addresses are remaining, and the rest of your seeds from the wallet", Run: wrap(walletseedscmd), } walletSendCmd = &cobra.Command{ Use: "send", Short: "Send either siacoins or siafunds to an address", Long: "Send either siacoins or siafunds to an address", // Run field is not set, as the load command itself is not a valid command. // A subcommand must be provided. } walletSendSiacoinsCmd = &cobra.Command{ Use: "siacoins [amount] [dest]", Short: "Send siacoins to an address", Long: `Send siacoins to an address. 'dest' must be a 76-byte hexadecimal address. 'amount' can be specified in units, e.g. 1.23KS. Run 'wallet --help' for a list of units. If no unit is supplied, hastings will be assumed. A miner fee of 10 SC is levied on all transactions.`, Run: wrap(walletsendsiacoinscmd), } walletSendSiafundsCmd = &cobra.Command{ Use: "siafunds [amount] [dest]", Short: "Send siafunds", Long: `Send siafunds to an address, and transfer the claim siacoins to your wallet. Run 'wallet send --help' to see a list of available units.`, Run: wrap(walletsendsiafundscmd), } walletBalanceCmd = &cobra.Command{ Use: "balance", Short: "View wallet balance", Long: "View wallet balance, including confirmed and unconfirmed siacoins and siafunds.", Run: wrap(walletbalancecmd), } walletTransactionsCmd = &cobra.Command{ Use: "transactions", Short: "View transactions", Long: "View transactions related to addresses spendable by the wallet, providing a net flow of siacoins and siafunds for each transaction", Run: wrap(wallettransactionscmd), } walletUnlockCmd = &cobra.Command{ Use: `unlock`, Short: "Unlock the wallet", Long: "Decrypt and load the wallet into memory", Run: wrap(walletunlockcmd), } ) const askPasswordText = "We need to encrypt the new data using the current wallet password, please provide: " // walletaddresscmd fetches a new address from the wallet that will be able to // receive coins. func walletaddresscmd() { addr := new(api.WalletAddressGET) err := getAPI("/wallet/address", addr) if err != nil { die("Could not generate new address:", err) } fmt.Printf("Created new address: %s\n", addr.Address) } // walletaddressescmd fetches the list of addresses that the wallet knows. func walletaddressescmd() { addrs := new(api.WalletAddressesGET) err := getAPI("/wallet/addresses", addrs) if err != nil { die("Failed to fetch addresses:", err) } for _, addr := range addrs.Addresses { fmt.Println(addr) } } // walletinitcmd encrypts the wallet with the given password func walletinitcmd() { var er api.WalletInitPOST qs := fmt.Sprintf("dictionary=%s", "english") if initPassword { password, err := speakeasy.Ask("Wallet password: ") if err != nil { die("Reading password failed:", err) } qs += fmt.Sprintf("&encryptionpassword=%s", password) } err := postResp("/wallet/init", qs, &er) if err != nil { die("Error when encrypting wallet:", err) } fmt.Printf("Recovery seed:\n%s\n\n", er.PrimarySeed) if initPassword { fmt.Printf("Wallet encrypted with given password\n") } else { fmt.Printf("Wallet encrypted with password:\n%s\n", er.PrimarySeed) } } // walletload033xcmd loads a v0.3.3.x wallet into the current wallet. func walletload033xcmd(source string) { password, err := speakeasy.Ask(askPasswordText) if err != nil { die("Reading password failed:", err) } qs := fmt.Sprintf("source=%s&encryptionpassword=%s", abs(source), password) err = post("/wallet/033x", qs) if err != nil { die("Loading wallet failed:", err) } fmt.Println("Wallet loading successful.") } // walletloadseedcmd adds a seed to the wallet's list of seeds func walletloadseedcmd() { seed, err := speakeasy.Ask("New seed: ") if err != nil { die("Reading seed failed:", err) } password, err := speakeasy.Ask(askPasswordText) if err != nil { die("Reading password failed:", err) } qs := fmt.Sprintf("encryptionpassword=%s&seed=%s&dictionary=%s", password, seed, "english") err = post("/wallet/seed", qs) if err != nil { die("Could not add seed:", err) } fmt.Println("Added Key") } // walletloadsiagcmd loads a siag key set into the wallet. func walletloadsiagcmd(keyfiles string) { password, err := speakeasy.Ask(askPasswordText) if err != nil { die("Reading password failed:", err) } qs := fmt.Sprintf("keyfiles=%s&encryptionpassword=%s", keyfiles, password) err = post("/wallet/siagkey", qs) if err != nil { die("Loading siag key failed:", err) } fmt.Println("Wallet loading successful.") } // walletlockcmd locks the wallet func walletlockcmd() { err := post("/wallet/lock", "") if err != nil { die("Could not lock wallet:", err) } } // walletseedcmd returns the current seed { func walletseedscmd() { var seedInfo api.WalletSeedsGET err := getAPI("/wallet/seeds", &seedInfo) if err != nil { die("Error retrieving the current seed:", err) } fmt.Printf("Primary Seed: %s\n"+ "Addresses Remaining %d\n"+ "All Seeds:\n", seedInfo.PrimarySeed, seedInfo.AddressesRemaining) for _, seed := range seedInfo.AllSeeds { fmt.Println(seed) } } // walletsendsiacoinscmd sends siacoins to a destination address. func walletsendsiacoinscmd(amount, dest string) { hastings, err := parseCurrency(amount) if err != nil { die("Could not parse amount:", err) } err = post("/wallet/siacoins", fmt.Sprintf("amount=%s&destination=%s", hastings, dest)) if err != nil { die("Could not send siacoins:", err) } fmt.Printf("Sent %s hastings to %s\n", hastings, dest) } // walletsendsiafundscmd sends siafunds to a destination address. func walletsendsiafundscmd(amount, dest string) { err := post("/wallet/siafunds", fmt.Sprintf("amount=%s&destination=%s", amount, dest)) if err != nil { die("Could not send siafunds:", err) } fmt.Printf("Sent %s siafunds to %s\n", amount, dest) } // walletbalancecmd retrieves and displays information about the wallet. func walletbalancecmd() { status := new(api.WalletGET) err := getAPI("/wallet", status) if err != nil { die("Could not get wallet status:", err) } encStatus := "Unencrypted" if status.Encrypted { encStatus = "Encrypted" } if !status.Unlocked { fmt.Printf(`Wallet status: %v, Locked Unlock the wallet to view balance `, encStatus) return } unconfirmedBalance := status.ConfirmedSiacoinBalance.Add(status.UnconfirmedIncomingSiacoins).Sub(status.UnconfirmedOutgoingSiacoins) var delta string if unconfirmedBalance.Cmp(status.ConfirmedSiacoinBalance) >= 0 { delta = "+" + currencyUnits(unconfirmedBalance.Sub(status.ConfirmedSiacoinBalance)) } else { delta = "-" + currencyUnits(status.ConfirmedSiacoinBalance.Sub(unconfirmedBalance)) } fmt.Printf(`Wallet status: %s, Unlocked Confirmed Balance: %v Unconfirmed Delta: %v Exact: %v H Siafunds: %v SF Siafund Claims: %v H `, encStatus, currencyUnits(status.ConfirmedSiacoinBalance), delta, status.ConfirmedSiacoinBalance, status.SiafundBalance, status.SiacoinClaimBalance) } // wallettransactionscmd lists all of the transactions related to the wallet, // providing a net flow of siacoins and siafunds for each. func wallettransactionscmd() { wtg := new(api.WalletTransactionsGET) err := getAPI("/wallet/transactions?startheight=0&endheight=10000000", wtg) if err != nil { die("Could not fetch transaction history:", err) } fmt.Println(" [height] [transaction id] [net siacoins] [net siafunds]") txns := append(wtg.ConfirmedTransactions, wtg.UnconfirmedTransactions...) for _, txn := range txns { // Determine the number of outgoing siacoins and siafunds. var outgoingSiacoins types.Currency var outgoingSiafunds types.Currency for _, input := range txn.Inputs { if input.FundType == types.SpecifierSiacoinInput && input.WalletAddress { outgoingSiacoins = outgoingSiacoins.Add(input.Value) } if input.FundType == types.SpecifierSiafundInput && input.WalletAddress { outgoingSiafunds = outgoingSiafunds.Add(input.Value) } } // Determine the number of incoming siacoins and siafunds. var incomingSiacoins types.Currency var incomingSiafunds types.Currency for _, output := range txn.Outputs { if output.FundType == types.SpecifierMinerPayout { incomingSiacoins = incomingSiacoins.Add(output.Value) } if output.FundType == types.SpecifierSiacoinOutput && output.WalletAddress { incomingSiacoins = incomingSiacoins.Add(output.Value) } if output.FundType == types.SpecifierSiafundOutput && output.WalletAddress { incomingSiafunds = incomingSiafunds.Add(output.Value) } } // Convert the siacoins to a float. incomingSiacoinsFloat, _ := new(big.Rat).SetFrac(incomingSiacoins.Big(), types.SiacoinPrecision.Big()).Float64() outgoingSiacoinsFloat, _ := new(big.Rat).SetFrac(outgoingSiacoins.Big(), types.SiacoinPrecision.Big()).Float64() // Print the results. if txn.ConfirmationHeight < 1e9 { fmt.Printf("%12v", txn.ConfirmationHeight) } else { fmt.Printf(" unconfirmed") } fmt.Printf("%67v%15.2f SC", txn.TransactionID, incomingSiacoinsFloat-outgoingSiacoinsFloat) // For siafunds, need to avoid having a negative types.Currency. if incomingSiafunds.Cmp(outgoingSiafunds) >= 0 { fmt.Printf("%14v SF\n", incomingSiafunds.Sub(outgoingSiafunds)) } else { fmt.Printf("-%14v SF\n", outgoingSiafunds.Sub(incomingSiafunds)) } } } // walletunlockcmd unlocks a saved wallet func walletunlockcmd() { password, err := speakeasy.Ask("Wallet password: ") if err != nil { die("Reading password failed:", err) } fmt.Println("Unlocking the wallet. This may take several minutes...") qs := fmt.Sprintf("encryptionpassword=%s&dictonary=%s", password, "english") err = post("/wallet/unlock", qs) if err != nil { die("Could not unlock wallet:", err) } fmt.Println("Wallet unlocked") } Sia-1.0.4/siad/000077500000000000000000000000001302033070500131415ustar00rootroot00000000000000Sia-1.0.4/siad/daemon.go000066400000000000000000000211141302033070500147320ustar00rootroot00000000000000package main import ( "errors" "fmt" "os" "os/signal" "path/filepath" "strconv" "strings" "time" "github.com/NebulousLabs/Sia/api" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/modules/consensus" "github.com/NebulousLabs/Sia/modules/explorer" "github.com/NebulousLabs/Sia/modules/gateway" "github.com/NebulousLabs/Sia/modules/host" "github.com/NebulousLabs/Sia/modules/miner" "github.com/NebulousLabs/Sia/modules/renter" "github.com/NebulousLabs/Sia/modules/transactionpool" "github.com/NebulousLabs/Sia/modules/wallet" "github.com/NebulousLabs/Sia/profile" "github.com/bgentry/speakeasy" "github.com/spf13/cobra" ) // verifyAPISecurity checks that the security values are consistent with a // sane, secure system. func verifyAPISecurity(config Config) error { // Make sure that only the loopback address is allowed unless the // --disable-api-security flag has been used. if !config.Siad.AllowAPIBind { addr := modules.NetAddress(config.Siad.APIaddr) if !addr.IsLoopback() { if addr.Host() == "" { return fmt.Errorf("a blank host will listen on all interfaces, did you mean localhost:%v?\nyou must pass --disable-api-security to bind Siad to a non-localhost address", addr.Port()) } return errors.New("you must pass --disable-api-security to bind Siad to a non-localhost address") } return nil } // If the --disable-api-security flag is used, enforce that // --authenticate-api must also be used. if config.Siad.AllowAPIBind && !config.Siad.AuthenticateAPI { return errors.New("cannot use --disable-api-security without setting an api password") } return nil } // processNetAddr adds a ':' to a bare integer, so that it is a proper port // number. func processNetAddr(addr string) string { _, err := strconv.Atoi(addr) if err == nil { return ":" + addr } return addr } // processModules makes the modules string lowercase to make checking if a // module in the string easier, and returns an error if the string contains an // invalid module character. func processModules(modules string) (string, error) { modules = strings.ToLower(modules) validModules := "cghmrtwe" invalidModules := modules for _, m := range validModules { invalidModules = strings.Replace(invalidModules, string(m), "", 1) } if len(invalidModules) > 0 { return "", errors.New("Unable to parse --modules flag, unrecognized or duplicate modules: " + invalidModules) } return modules, nil } // processConfig checks the configuration values and performs cleanup on // incorrect-but-allowed values. func processConfig(config Config) (Config, error) { var err1 error config.Siad.APIaddr = processNetAddr(config.Siad.APIaddr) config.Siad.RPCaddr = processNetAddr(config.Siad.RPCaddr) config.Siad.HostAddr = processNetAddr(config.Siad.HostAddr) config.Siad.Modules, err1 = processModules(config.Siad.Modules) err2 := verifyAPISecurity(config) err := build.JoinErrors([]error{err1, err2}, ", and ") if err != nil { return Config{}, err } return config, nil } // startDaemon uses the config parameters to initialize Sia modules and start // siad. func startDaemon(config Config) (err error) { // Prompt user for API password. if config.Siad.AuthenticateAPI { config.APIPassword, err = speakeasy.Ask("Enter API password: ") if err != nil { return err } if config.APIPassword == "" { return errors.New("password cannot be blank") } } // Process the config variables after they are parsed by cobra. config, err = processConfig(config) if err != nil { return err } // Print a startup message. fmt.Println("Loading...") loadStart := time.Now() // Create the server and start serving daemon routes immediately. fmt.Printf("(0/%d) Loading siad...\n", len(config.Siad.Modules)) srv, err := NewServer(config.Siad.APIaddr, config.Siad.RequiredUserAgent, config.APIPassword) if err != nil { return err } servErrs := make(chan error) go func() { servErrs <- srv.Serve() }() // Initialize the Sia modules i := 0 var g modules.Gateway if strings.Contains(config.Siad.Modules, "g") { i++ fmt.Printf("(%d/%d) Loading gateway...\n", i, len(config.Siad.Modules)) g, err = gateway.New(config.Siad.RPCaddr, !config.Siad.NoBootstrap, filepath.Join(config.Siad.SiaDir, modules.GatewayDir)) if err != nil { return err } defer func() { fmt.Println("Closing gateway...") err := g.Close() if err != nil { fmt.Println("Error during gateway shutdown:", err) } }() } var cs modules.ConsensusSet if strings.Contains(config.Siad.Modules, "c") { i++ fmt.Printf("(%d/%d) Loading consensus...\n", i, len(config.Siad.Modules)) cs, err = consensus.New(g, !config.Siad.NoBootstrap, filepath.Join(config.Siad.SiaDir, modules.ConsensusDir)) if err != nil { return err } defer func() { fmt.Println("Closing consensus set...") err := cs.Close() if err != nil { fmt.Println("Error during consensus set shutdown:", err) } }() } var e modules.Explorer if strings.Contains(config.Siad.Modules, "e") { i++ fmt.Printf("(%d/%d) Loading explorer...\n", i, len(config.Siad.Modules)) e, err = explorer.New(cs, filepath.Join(config.Siad.SiaDir, modules.ExplorerDir)) if err != nil { return err } defer func() { fmt.Println("Closing explorer...") err := e.Close() if err != nil { fmt.Println("Error during explorer shutdown:", err) } }() } var tpool modules.TransactionPool if strings.Contains(config.Siad.Modules, "t") { i++ fmt.Printf("(%d/%d) Loading transaction pool...\n", i, len(config.Siad.Modules)) tpool, err = transactionpool.New(cs, g, filepath.Join(config.Siad.SiaDir, modules.TransactionPoolDir)) if err != nil { return err } defer func() { fmt.Println("Closing transaction pool...") err := tpool.Close() if err != nil { fmt.Println("Error during transaction pool shutdown:", err) } }() } var w modules.Wallet if strings.Contains(config.Siad.Modules, "w") { i++ fmt.Printf("(%d/%d) Loading wallet...\n", i, len(config.Siad.Modules)) w, err = wallet.New(cs, tpool, filepath.Join(config.Siad.SiaDir, modules.WalletDir)) if err != nil { return err } defer func() { fmt.Println("Closing wallet...") err := w.Close() if err != nil { fmt.Println("Error during wallet shutdown:", err) } }() } var m modules.Miner if strings.Contains(config.Siad.Modules, "m") { i++ fmt.Printf("(%d/%d) Loading miner...\n", i, len(config.Siad.Modules)) m, err = miner.New(cs, tpool, w, filepath.Join(config.Siad.SiaDir, modules.MinerDir)) if err != nil { return err } defer func() { fmt.Println("Closing miner...") err := m.Close() if err != nil { fmt.Println("Error during miner shutdown:", err) } }() } var h modules.Host if strings.Contains(config.Siad.Modules, "h") { i++ fmt.Printf("(%d/%d) Loading host...\n", i, len(config.Siad.Modules)) h, err = host.New(cs, tpool, w, config.Siad.HostAddr, filepath.Join(config.Siad.SiaDir, modules.HostDir)) if err != nil { return err } defer func() { fmt.Println("Closing host...") err := h.Close() if err != nil { fmt.Println("Error during host shutdown:", err) } }() } var r modules.Renter if strings.Contains(config.Siad.Modules, "r") { i++ fmt.Printf("(%d/%d) Loading renter...\n", i, len(config.Siad.Modules)) r, err = renter.New(cs, w, tpool, filepath.Join(config.Siad.SiaDir, modules.RenterDir)) if err != nil { return err } defer func() { fmt.Println("Closing renter...") err := r.Close() if err != nil { fmt.Println("Error during renter shutdown:", err) } }() } // Create the Sia API a := api.New( config.Siad.RequiredUserAgent, config.APIPassword, cs, e, g, h, m, r, tpool, w, ) // connect the API to the server srv.mux.Handle("/", a) // stop the server if a kill signal is caught sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt, os.Kill) go func() { <-sigChan fmt.Println("\rCaught stop signal, quitting...") srv.Close() }() // Print a 'startup complete' message. startupTime := time.Since(loadStart) fmt.Println("Finished loading in", startupTime.Seconds(), "seconds") err = <-servErrs if err != nil { build.Critical(err) } return nil } // startDaemonCmd is a passthrough function for startDaemon. func startDaemonCmd(cmd *cobra.Command, _ []string) { // Create the profiling directory if profiling is enabled. if globalConfig.Siad.Profile || build.DEBUG { go profile.StartContinuousProfile(globalConfig.Siad.ProfileDir) } // Start siad. startDaemon will only return when it is shutting down. err := startDaemon(globalConfig) if err != nil { die(err) } } Sia-1.0.4/siad/daemon_test.go000066400000000000000000000117231302033070500157760ustar00rootroot00000000000000package main import ( "testing" ) // TestUnitProcessNetAddr probes the 'processNetAddr' function. func TestUnitProcessNetAddr(t *testing.T) { testVals := struct { inputs []string expectedOutputs []string }{ inputs: []string{"9980", ":9980", "localhost:9980", "test.com:9980", "192.168.14.92:9980"}, expectedOutputs: []string{":9980", ":9980", "localhost:9980", "test.com:9980", "192.168.14.92:9980"}, } for i, input := range testVals.inputs { output := processNetAddr(input) if output != testVals.expectedOutputs[i] { t.Error("unexpected result", i) } } } // TestUnitProcessModules tests that processModules correctly processes modules // passed to the -M / --modules flag. func TestUnitProcessModules(t *testing.T) { // Test valid modules. testVals := []struct { in string out string }{ {"cghmrtwe", "cghmrtwe"}, {"CGHMRTWE", "cghmrtwe"}, {"c", "c"}, {"g", "g"}, {"h", "h"}, {"m", "m"}, {"r", "r"}, {"t", "t"}, {"w", "w"}, {"e", "e"}, {"C", "c"}, {"G", "g"}, {"H", "h"}, {"M", "m"}, {"R", "r"}, {"T", "t"}, {"W", "w"}, {"E", "e"}, } for _, testVal := range testVals { out, err := processModules(testVal.in) if err != nil { t.Error("processModules failed with error:", err) } if out != testVal.out { t.Errorf("processModules returned incorrect modules: expected %s, got %s\n", testVal.out, out) } } // Test invalid modules. invalidModules := []string{"abdfijklnopqsuvxyz", "cghmrtwez", "cz", "z", "cc", "ccz", "ccm", "cmm", "ccmm"} for _, invalidModule := range invalidModules { _, err := processModules(invalidModule) if err == nil { t.Error("processModules didn't error on invalid module:", invalidModule) } } } // TestUnitProcessConfig probes the 'processConfig' function. func TestUnitProcessConfig(t *testing.T) { // Test valid configs. testVals := struct { inputs [][]string expectedOutputs [][]string }{ inputs: [][]string{ {"localhost:9980", "localhost:9981", "localhost:9982", "cghmrtwe"}, {"localhost:9980", "localhost:9981", "localhost:9982", "CGHMRTWE"}, }, expectedOutputs: [][]string{ {"localhost:9980", "localhost:9981", "localhost:9982", "cghmrtwe"}, {"localhost:9980", "localhost:9981", "localhost:9982", "cghmrtwe"}, }, } var config Config for i := range testVals.inputs { config.Siad.APIaddr = testVals.inputs[i][0] config.Siad.RPCaddr = testVals.inputs[i][1] config.Siad.HostAddr = testVals.inputs[i][2] config, err := processConfig(config) if err != nil { t.Error("processConfig failed with error:", err) } if config.Siad.APIaddr != testVals.expectedOutputs[i][0] { t.Error("processing failure at check", i, 0) } if config.Siad.RPCaddr != testVals.expectedOutputs[i][1] { t.Error("processing failure at check", i, 1) } if config.Siad.HostAddr != testVals.expectedOutputs[i][2] { t.Error("processing failure at check", i, 2) } } // Test invalid configs. invalidModule := "z" config.Siad.Modules = invalidModule _, err := processConfig(config) if err == nil { t.Error("processModules didn't error on invalid module:", invalidModule) } } // TestVerifyAPISecurity checks that the verifyAPISecurity function is // correctly banning the use of a non-loopback address without the // --disable-security flag, and that the --disable-security flag cannot be used // without an api password. func TestVerifyAPISecurity(t *testing.T) { // Check that the loopback address is accepted when security is enabled. var securityOnLoopback Config securityOnLoopback.Siad.APIaddr = "127.0.0.1:9980" err := verifyAPISecurity(securityOnLoopback) if err != nil { t.Error("loopback + securityOn was rejected") } // Check that the blank address is rejected when security is enabled. var securityOnBlank Config securityOnBlank.Siad.APIaddr = ":9980" err = verifyAPISecurity(securityOnBlank) if err == nil { t.Error("blank + securityOn was accepted") } // Check that a public hostname is rejected when security is enabled. var securityOnPublic Config securityOnPublic.Siad.APIaddr = "sia.tech:9980" err = verifyAPISecurity(securityOnPublic) if err == nil { t.Error("public + securityOn was accepted") } // Check that a public hostname is rejected when security is disabled and // there is no api password. var securityOffPublic Config securityOffPublic.Siad.APIaddr = "sia.tech:9980" securityOffPublic.Siad.AllowAPIBind = true err = verifyAPISecurity(securityOffPublic) if err == nil { t.Error("public + securityOff was accepted without authentication") } // Check that a public hostname is accepted when security is disabled and // there is an api password. var securityOffPublicAuthenticated Config securityOffPublicAuthenticated.Siad.APIaddr = "sia.tech:9980" securityOffPublicAuthenticated.Siad.AllowAPIBind = true securityOffPublicAuthenticated.Siad.AuthenticateAPI = true err = verifyAPISecurity(securityOffPublicAuthenticated) if err != nil { t.Error("public + securityOff with authentication was rejected:", err) } } Sia-1.0.4/siad/main.go000066400000000000000000000143371302033070500144240ustar00rootroot00000000000000package main import ( "fmt" "os" "github.com/spf13/cobra" "github.com/NebulousLabs/Sia/build" ) var ( // globalConfig is used by the cobra package to fill out the configuration // variables. globalConfig Config ) // exit codes // inspired by sysexits.h const ( exitCodeGeneral = 1 // Not in sysexits.h, but is standard practice. exitCodeUsage = 64 // EX_USAGE in sysexits.h ) // The Config struct contains all configurable variables for siad. It is // compatible with gcfg. type Config struct { // The APIPassword is input by the user after the daemon starts up, if the // --authenticate-api flag is set. APIPassword string // The Siad variables are referenced directly by cobra, and are set // according to the flags. Siad struct { APIaddr string RPCaddr string HostAddr string AllowAPIBind bool Modules string NoBootstrap bool RequiredUserAgent string AuthenticateAPI bool Profile bool ProfileDir string SiaDir string } } // die prints its arguments to stderr, then exits the program with the default // error code. func die(args ...interface{}) { fmt.Fprintln(os.Stderr, args...) os.Exit(exitCodeGeneral) } // versionCmd is a cobra command that prints the version of siad. func versionCmd(*cobra.Command, []string) { switch build.Release { case "dev": fmt.Println("Sia Daemon v" + build.Version + "-dev") case "standard": fmt.Println("Sia Daemon v" + build.Version) case "testing": fmt.Println("Sia Daemon v" + build.Version + "-testing") default: fmt.Println("Sia Daemon v" + build.Version + "-???") } } // modulesCmd is a cobra command that prints help info about modules. func modulesCmd(*cobra.Command, []string) { fmt.Println(`Use the -M or --modules flag to only run specific modules. Modules are independent components of Sia. This flag should only be used by developers or people who want to reduce overhead from unused modules. Modules are specified by their first letter. If the -M or --modules flag is not specified the default modules are run. The default modules are: gateway, consensus set, host, miner, renter, transaction pool, wallet This is equivalent to: siad -M cghmrtw Below is a list of all the modules available. Gateway (g): The gateway maintains a peer to peer connection to the network and enables other modules to perform RPC calls on peers. The gateway is required by all other modules. Example: siad -M g Consensus Set (c): The consensus set manages everything related to consensus and keeps the blockchain in sync with the rest of the network. The consensus set requires the gateway. Example: siad -M gc Transaction Pool (t): The transaction pool manages unconfirmed transactions. The transaction pool requires the consensus set. Example: siad -M gct Wallet (w): The wallet stores and manages siacoins and siafunds. The wallet requires the consensus set and transaction pool. Example: siad -M gctw Renter (r): The renter manages the user's files on the network. The renter requires the consensus set, transaction pool, and wallet. Example: siad -M gctwr Host (h): The host provides storage from local disks to the network. The host negotiates file contracts with remote renters to earn money for storing other users' files. The host requires the consensus set, transaction pool, and wallet. Example: siad -M gctwh Miner (m): The miner provides a basic CPU mining implementation as well as an API for external miners to use. The miner requires the consensus set, transaction pool, and wallet. Example: siad -M gctwm Explorer (e): The explorer provides statistics about the blockchain and can be queried for information about specific transactions or other objects on the blockchain. The explorer requires the consenus set. Example: siad -M gce`) } // main establishes a set of commands and flags using the cobra package. func main() { root := &cobra.Command{ Use: os.Args[0], Short: "Sia Daemon v" + build.Version, Long: "Sia Daemon v" + build.Version, Run: startDaemonCmd, } root.AddCommand(&cobra.Command{ Use: "version", Short: "Print version information", Long: "Print version information about the Sia Daemon", Run: versionCmd, }) root.AddCommand(&cobra.Command{ Use: "modules", Short: "List available modules for use with -M, --modules flag", Long: "List available modules for use with -M, --modules flag and their uses", Run: modulesCmd, }) // Set default values, which have the lowest priority. root.Flags().StringVarP(&globalConfig.Siad.RequiredUserAgent, "agent", "", "Sia-Agent", "required substring for the user agent") root.Flags().StringVarP(&globalConfig.Siad.HostAddr, "host-addr", "", ":9982", "which port the host listens on") root.Flags().StringVarP(&globalConfig.Siad.ProfileDir, "profile-directory", "", "profiles", "location of the profiling directory") root.Flags().StringVarP(&globalConfig.Siad.APIaddr, "api-addr", "", "localhost:9980", "which host:port the API server listens on") root.Flags().StringVarP(&globalConfig.Siad.SiaDir, "sia-directory", "d", "", "location of the sia directory") root.Flags().BoolVarP(&globalConfig.Siad.NoBootstrap, "no-bootstrap", "", false, "disable bootstrapping on this run") root.Flags().BoolVarP(&globalConfig.Siad.Profile, "profile", "", false, "enable profiling") root.Flags().StringVarP(&globalConfig.Siad.RPCaddr, "rpc-addr", "", ":9981", "which port the gateway listens on") root.Flags().StringVarP(&globalConfig.Siad.Modules, "modules", "M", "cghmrtw", "enabled modules, see 'siad modules' for more info") root.Flags().BoolVarP(&globalConfig.Siad.AuthenticateAPI, "authenticate-api", "", false, "enable API password protection") root.Flags().BoolVarP(&globalConfig.Siad.AllowAPIBind, "disable-api-security", "", false, "allow siad to listen on a non-localhost address (DANGEROUS)") // Parse cmdline flags, overwriting both the default values and the config // file values. if err := root.Execute(); err != nil { // Since no commands return errors (all commands set Command.Run instead of // Command.RunE), Command.Execute() should only return an error on an // invalid command or flag. Therefore Command.Usage() was called (assuming // Command.SilenceUsage is false) and we should exit with exitCodeUsage. os.Exit(exitCodeUsage) } } Sia-1.0.4/siad/server.go000066400000000000000000000317771302033070500150150ustar00rootroot00000000000000package main import ( "archive/zip" "bytes" "encoding/json" "errors" "fmt" "io" "io/ioutil" "math/big" "net" "net/http" "path" "path/filepath" "runtime" "sort" "strings" "github.com/NebulousLabs/Sia/api" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/types" "github.com/inconshreveable/go-update" "github.com/julienschmidt/httprouter" "github.com/kardianos/osext" ) var errEmptyUpdateResponse = errors.New("API call to https://api.github.com/repos/NebulousLabs/Sia/releases/latest is returning an empty response") type ( // Server creates and serves a HTTP server that offers communication with a // Sia API. Server struct { httpServer *http.Server mux *http.ServeMux listener net.Listener } // SiaConstants is a struct listing all of the constants in use. SiaConstants struct { GenesisTimestamp types.Timestamp `json:"genesistimestamp"` BlockSizeLimit uint64 `json:"blocksizelimit"` BlockFrequency types.BlockHeight `json:"blockfrequency"` TargetWindow types.BlockHeight `json:"targetwindow"` MedianTimestampWindow uint64 `json:"mediantimestampwindow"` FutureThreshold types.Timestamp `json:"futurethreshold"` SiafundCount types.Currency `json:"siafundcount"` SiafundPortion *big.Rat `json:"siafundportion"` MaturityDelay types.BlockHeight `json:"maturitydelay"` InitialCoinbase uint64 `json:"initialcoinbase"` MinimumCoinbase uint64 `json:"minimumcoinbase"` RootTarget types.Target `json:"roottarget"` RootDepth types.Target `json:"rootdepth"` MaxAdjustmentUp *big.Rat `json:"maxadjustmentup"` MaxAdjustmentDown *big.Rat `json:"maxadjustmentdown"` SiacoinPrecision types.Currency `json:"siacoinprecision"` } DaemonVersion struct { Version string `json:"version"` } // UpdateInfo indicates whether an update is available, and to what // version. UpdateInfo struct { Available bool `json:"available"` Version string `json:"version"` } // githubRelease represents some of the JSON returned by the GitHub release API // endpoint. Only the fields relevant to updating are included. githubRelease struct { TagName string `json:"tag_name"` Assets []struct { Name string `json:"name"` DownloadURL string `json:"browser_download_url"` } `json:"assets"` } ) const ( // The developer key is used to sign updates and other important Sia- // related information. developerKey = `-----BEGIN PUBLIC KEY----- MIIEIjANBgkqhkiG9w0BAQEFAAOCBA8AMIIECgKCBAEAsoQHOEU6s/EqMDtw5HvA YPTUaBgnviMFbG3bMsRqSCD8ug4XJYh+Ik6WP0xgq+OPDehPiaXK8ghAtBiW1EJK mBRwlABXAzREZg8wRfG4l8Zj6ckAPJOgLn0jobXy6/SCQ+jZSWh4Y8DYr+LA3Mn3 EOga7Jvhpc3fTZ232GBGJ1BobuNfRfYmwxSphv+T4vzIA3JUjVfa8pYZGIjh5XbJ 5M8Lef0Xa9eqr6lYm5kQoOIXeOW56ImqI2BKg/I9NGw9phSPbwaFfy1V2kfHp5Xy DtKnyj/O9zDi+qUKjoIivnEoV+3DkioHUWv7Fpf7yx/9cPyckwvaBsTd9Cfp4uBx qJ5Qyv69VZQiD6DikNwgzjGbIjiLwfTObhInKZUoYl48yzgkR80ja5TW0SoidNvO 4WTbWcLolOl522VarTs7wlgbq0Ad7yrNVnHzo447v2iT20ILH2oeAcZqvpcvRmTl U6uKoaVmBH3D3Y19dPluOjK53BrqfQ5L8RFli2wEJktPsi5fUTd4UI9BgnUieuDz S7h/VH9bv9ZVvyjpu/uVjdvaikT3zbIy9J6wS6uE5qPLPhI4B9HgbrQ03muDGpql gZrMiL3GdYrBiqpIbaWHfM0eMWEK3ZScUdtCgUXMMrkvaUJ4g9wEgbONFVVOMIV+ YubIuzBFqug6WyxN/EAM/6Fss832AwVPcYM0NDTVGVdVplLMdN8YNjrYuaPngBCG e8QaTWtHzLujyBIkVdAHqfkRS65jp7JLLMx7jUA74/E/v+0cNew3Y1p2gt3iQH8t w93xn9IPUfQympc4h3KerP/Yn6P/qAh68jQkOiMMS+VbCq/BOn8Q3GbR+8rQ8dmk qVoGA7XrPQ6bymKBTghk2Ek+ZjxrpAoj0xYoYyzWf0kuxeOT8kAjlLLmfQ8pm75S QHLqH49FyfeETIU02rkw2oMOX/EYdJzZukHuouwbpKSElpRx+xTnaSemMJo+U7oX xVjma3Zynh9w12abnFWkZKtrxwXv7FCSzb0UZmMWUqWzCS03Rrlur21jp4q2Wl71 Vt92xe5YbC/jbh386F1e/qGq6p+D1AmBynIpp/HE6fPsc9LWgJDDkREZcp7hthGW IdYPeP3CesFHnsZMueZRib0i7lNUkBSRneO1y/C9poNv1vOeTCNEE0jvhp/XOJuc yCQtrUSNALsvm7F+bnwP2F7K34k7MOlOgnTGqCqW+9WwBcjR44B0HI+YERCcRmJ8 krBuVo9OBMV0cYBWpjo3UI9j3lHESCYhLnCz7SPap7C1yORc2ydJh+qjKqdLBHom t+JydcdJLbIG+kb3jB9QIIu5A4TlSGlHV6ewtxIWLS1473jEkITiVTt0Y5k+VLfW bwIDAQAB -----END PUBLIC KEY-----` ) // version returns the version number of a non-LTS release. This assumes that // tag names will always be of the form "vX.Y.Z". func (r *githubRelease) version() string { return strings.TrimPrefix(r.TagName, "v") } // byVersion sorts non-LTS releases by their version string, placing the highest // version number first. type byVersion []githubRelease func (rs byVersion) Len() int { return len(rs) } func (rs byVersion) Swap(i, j int) { rs[i], rs[j] = rs[j], rs[i] } func (rs byVersion) Less(i, j int) bool { // we want the higher version number to reported as "less" so that it is // placed first in the slice return build.VersionCmp(rs[i].version(), rs[j].version()) >= 0 } // latestRelease returns the latest non-LTS release, given a set of arbitrary // releases. func latestRelease(releases []githubRelease) (githubRelease, error) { // filter the releases to exclude LTS releases nonLTS := releases[:0] for _, r := range releases { if !strings.Contains(r.TagName, "lts") && build.IsVersion(r.version()) { nonLTS = append(nonLTS, r) } } // sort by version sort.Sort(byVersion(nonLTS)) // return the latest release if len(nonLTS) == 0 { return githubRelease{}, errEmptyUpdateResponse } return nonLTS[0], nil } // fetchLatestRelease returns metadata about the most recent non-LTS GitHub // release. func fetchLatestRelease() (githubRelease, error) { req, err := http.NewRequest("GET", "https://api.github.com/repos/NebulousLabs/Sia/releases", nil) if err != nil { return githubRelease{}, err } req.Header.Set("Accept", "application/vnd.github.v3+json") resp, err := http.DefaultClient.Do(req) if err != nil { return githubRelease{}, err } defer resp.Body.Close() var releases []githubRelease err = json.NewDecoder(resp.Body).Decode(&releases) if err != nil { return githubRelease{}, err } return latestRelease(releases) } // updateToRelease updates siad and siac to the release specified. siac is // assumed to be in the same folder as siad. func updateToRelease(release githubRelease) error { updateOpts := update.Options{ Verifier: update.NewRSAVerifier(), } err := updateOpts.SetPublicKeyPEM([]byte(developerKey)) if err != nil { // should never happen return err } binaryFolder, err := osext.ExecutableFolder() if err != nil { return err } // construct release filename releaseName := fmt.Sprintf("Sia-%s-%s-%s.zip", release.TagName, runtime.GOOS, runtime.GOARCH) // find release var downloadURL string for _, asset := range release.Assets { if asset.Name == releaseName { downloadURL = asset.DownloadURL break } } if downloadURL == "" { return errors.New("couldn't find download URL for " + releaseName) } // download release archive resp, err := http.Get(downloadURL) if err != nil { return err } // release should be small enough to store in memory (<10 MiB); use // LimitReader to ensure we don't download more than 32 MiB content, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1<<25)) resp.Body.Close() if err != nil { return err } r := bytes.NewReader(content) z, err := zip.NewReader(r, r.Size()) if err != nil { return err } // process zip, finding siad/siac binaries and signatures for _, binary := range []string{"siad", "siac"} { var binData io.ReadCloser var signature []byte var binaryName string // needed for TargetPath below for _, zf := range z.File { switch base := path.Base(zf.Name); base { case binary, binary + ".exe": binaryName = base binData, err = zf.Open() if err != nil { return err } defer binData.Close() case binary + ".sig", binary + ".exe.sig": sigFile, err := zf.Open() if err != nil { return err } defer sigFile.Close() signature, err = ioutil.ReadAll(sigFile) if err != nil { return err } } } if binData == nil { return errors.New("could not find " + binary + " binary") } else if signature == nil { return errors.New("could not find " + binary + " signature") } // apply update updateOpts.Signature = signature updateOpts.TargetMode = 0775 // executable updateOpts.TargetPath = filepath.Join(binaryFolder, binaryName) err = update.Apply(binData, updateOpts) if err != nil { return err } } return nil } // daemonUpdateHandlerGET handles the API call that checks for an update. func (srv *Server) daemonUpdateHandlerGET(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { release, err := fetchLatestRelease() if err != nil { api.WriteError(w, api.Error{Message: "Failed to fetch latest release: " + err.Error()}, http.StatusInternalServerError) return } latestVersion := release.TagName[1:] // delete leading 'v' api.WriteJSON(w, UpdateInfo{ Available: build.VersionCmp(latestVersion, build.Version) > 0, Version: latestVersion, }) } // daemonUpdateHandlerPOST handles the API call that updates siad and siac. // There is no safeguard to prevent "updating" to the same release, so callers // should always check the latest version via daemonUpdateHandlerGET first. // TODO: add support for specifying version to update to. func (srv *Server) daemonUpdateHandlerPOST(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { release, err := fetchLatestRelease() if err != nil { api.WriteError(w, api.Error{Message: "Failed to fetch latest release: " + err.Error()}, http.StatusInternalServerError) return } err = updateToRelease(release) if err != nil { if rerr := update.RollbackError(err); rerr != nil { api.WriteError(w, api.Error{Message: "Serious error: Failed to rollback from bad update: " + rerr.Error()}, http.StatusInternalServerError) } else { api.WriteError(w, api.Error{Message: "Failed to apply update: " + err.Error()}, http.StatusInternalServerError) } return } api.WriteSuccess(w) } // debugConstantsHandler prints a json file containing all of the constants. func (srv *Server) daemonConstantsHandler(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { sc := SiaConstants{ GenesisTimestamp: types.GenesisTimestamp, BlockSizeLimit: types.BlockSizeLimit, BlockFrequency: types.BlockFrequency, TargetWindow: types.TargetWindow, MedianTimestampWindow: types.MedianTimestampWindow, FutureThreshold: types.FutureThreshold, SiafundCount: types.SiafundCount, SiafundPortion: types.SiafundPortion, MaturityDelay: types.MaturityDelay, InitialCoinbase: types.InitialCoinbase, MinimumCoinbase: types.MinimumCoinbase, RootTarget: types.RootTarget, RootDepth: types.RootDepth, MaxAdjustmentUp: types.MaxAdjustmentUp, MaxAdjustmentDown: types.MaxAdjustmentDown, SiacoinPrecision: types.SiacoinPrecision, } api.WriteJSON(w, sc) } // daemonVersionHandler handles the API call that requests the daemon's version. func (srv *Server) daemonVersionHandler(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { api.WriteJSON(w, DaemonVersion{Version: build.Version}) } // daemonStopHandler handles the API call to stop the daemon cleanly. func (srv *Server) daemonStopHandler(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { // can't write after we stop the server, so lie a bit. api.WriteSuccess(w) // need to flush the response before shutting down the server f, ok := w.(http.Flusher) if !ok { panic("Server does not support flushing") } f.Flush() if err := srv.Close(); err != nil { build.Critical(err) } } func (srv *Server) daemonHandler(password string) http.Handler { router := httprouter.New() router.GET("/daemon/constants", srv.daemonConstantsHandler) router.GET("/daemon/version", srv.daemonVersionHandler) router.GET("/daemon/update", srv.daemonUpdateHandlerGET) router.POST("/daemon/update", srv.daemonUpdateHandlerPOST) router.GET("/daemon/stop", api.RequirePassword(srv.daemonStopHandler, password)) return router } // NewServer creates a new net.http server listening on bindAddr. Only the // /daemon/ routes are registered by this func, additional routes can be // registered later by calling serv.mux.Handle. func NewServer(bindAddr, requiredUserAgent, requiredPassword string) (*Server, error) { // Create the listener for the server l, err := net.Listen("tcp", bindAddr) if err != nil { return nil, err } // Create the Server mux := http.NewServeMux() srv := &Server{ mux: mux, listener: l, httpServer: &http.Server{ Handler: mux, }, } // Register siad routes srv.mux.Handle("/daemon/", api.RequireUserAgent(srv.daemonHandler(requiredPassword), requiredUserAgent)) return srv, nil } func (srv *Server) Serve() error { // The server will run until an error is encountered or the listener is // closed, via either the Close method or the signal handling above. // Closing the listener will result in the benign error handled below. err := srv.httpServer.Serve(srv.listener) if err != nil && !strings.HasSuffix(err.Error(), "use of closed network connection") { return err } return nil } // Close closes the Server's listener, causing the HTTP server to shut down. func (srv *Server) Close() error { // Close the listener, which will cause Server.Serve() to return. if err := srv.listener.Close(); err != nil { return err } return nil } Sia-1.0.4/siad/server_test.go000066400000000000000000000027021302033070500160360ustar00rootroot00000000000000package main import "testing" // TestLatestRelease tests that the latestRelease function properly processes a // set of GitHub releases, returning the release with the highest version // number. func TestLatestRelease(t *testing.T) { tests := []struct { releases []githubRelease expectedTag string }{ { releases: []githubRelease{ {TagName: "lts-v1.0.4"}, {TagName: "v3.0.7"}, {TagName: "lts-v2.0.0"}, }, expectedTag: "v3.0.7", }, { releases: []githubRelease{ {TagName: "lts-v1.0.4"}, {TagName: "v3.0.7"}, {TagName: "v5.2.2"}, }, expectedTag: "v5.2.2", }, { releases: []githubRelease{ {TagName: "lts-v1.0.4"}, {TagName: "lts-v1.0.7"}, {TagName: "lts-v1.0.5"}, }, expectedTag: "", // no non-LTS versions }, { releases: []githubRelease{ {TagName: "v1.0.4"}, {TagName: "v1.0.7"}, {TagName: "v1.0.5"}, }, expectedTag: "v1.0.7", }, { releases: []githubRelease{ {TagName: "v1.0.4"}, {TagName: "v1.0.4.1"}, {TagName: "v1.0.4-patch1"}, }, expectedTag: "v1.0.4.1", // -patch is invalid }, { releases: []githubRelease{ {TagName: "abc"}, {TagName: "def"}, {TagName: "ghi"}, }, expectedTag: "", // invalid version strings }, } for i, test := range tests { r, _ := latestRelease(test.releases) if r.TagName != test.expectedTag { t.Errorf("test %v failed: expected %q, got %q", i, test.expectedTag, r.TagName) } } } Sia-1.0.4/sync/000077500000000000000000000000001302033070500131755ustar00rootroot00000000000000Sia-1.0.4/sync/lock.go000066400000000000000000000103471302033070500144610ustar00rootroot00000000000000package sync import ( "fmt" "os" "runtime" "sync" "time" ) // RWMutex provides locking functions, and an ability to detect and remove // deadlocks. type RWMutex struct { openLocks map[int]lockInfo openLocksCounter int openLocksMutex sync.Mutex callDepth int maxLockTime time.Duration mu sync.RWMutex } // lockInfo contains information about when and how a lock call was made. type lockInfo struct { // When the lock was called. lockTime time.Time // Whether it was a RLock or a Lock. read bool // Call stack of the caller. callingFiles []string callingLines []int } // New takes a maxLockTime and returns a lock. The lock will never stay locked // for more than maxLockTime, instead printing an error and unlocking after // maxLockTime has passed. func New(maxLockTime time.Duration, callDepth int) *RWMutex { rwm := &RWMutex{ openLocks: make(map[int]lockInfo), maxLockTime: maxLockTime, callDepth: callDepth, } go rwm.threadedDeadlockFinder() return rwm } // threadedDeadlockFinder occasionally freezes the mutexes and scans all open mutexes, // reporting any that have exceeded their time limit. func (rwm *RWMutex) threadedDeadlockFinder() { for { rwm.openLocksMutex.Lock() for id, info := range rwm.openLocks { // Check if the lock has been held for longer than 'maxLockTime'. if time.Now().Sub(info.lockTime) > rwm.maxLockTime { str := fmt.Sprintf("A lock was held for too long, id '%v'. Call stack:\n", id) for i := 0; i <= rwm.callDepth; i++ { str += fmt.Sprintf("\tFile: '%v:%v'\n", info.callingFiles[i], info.callingLines[i]) } os.Stderr.WriteString(str) os.Stderr.Sync() // Undo the deadlock and delete the entry from the map. if info.read { rwm.mu.RUnlock() } else { rwm.mu.Unlock() } delete(rwm.openLocks, id) } } rwm.openLocksMutex.Unlock() time.Sleep(rwm.maxLockTime) } } // safeLock is the generic function for doing safe locking. If the read flag is // set, then a readlock will be used, otherwise a lock will be used. func (rwm *RWMutex) safeLock(read bool) int { // Get the call stack. var li lockInfo li.read = read li.callingFiles = make([]string, rwm.callDepth+1) li.callingLines = make([]int, rwm.callDepth+1) for i := 0; i <= rwm.callDepth; i++ { _, li.callingFiles[i], li.callingLines[i], _ = runtime.Caller(2 + i) } // Lock the mutex. if read { rwm.mu.RLock() } else { rwm.mu.Lock() } // Safely register that a lock has been triggered. rwm.openLocksMutex.Lock() li.lockTime = time.Now() id := rwm.openLocksCounter rwm.openLocks[id] = li rwm.openLocksCounter++ rwm.openLocksMutex.Unlock() return id } // safeUnlock is the generic function for doing safe unlocking. If the lock had // to be removed because a deadlock was detected, an error is printed. func (rwm *RWMutex) safeUnlock(read bool, id int) { rwm.openLocksMutex.Lock() defer rwm.openLocksMutex.Unlock() // Check if a deadlock has been detected and fixed manually. _, exists := rwm.openLocks[id] if !exists { // Get the call stack. callingFiles := make([]string, rwm.callDepth+1) callingLines := make([]int, rwm.callDepth+1) for i := 0; i <= rwm.callDepth; i++ { _, callingFiles[i], callingLines[i], _ = runtime.Caller(2 + i) } fmt.Printf("A lock was held until deadlock, subsequent call to unlock failed. id '%v'. Call stack:\n", id) for i := 0; i <= rwm.callDepth; i++ { fmt.Printf("\tFile: '%v:%v'\n", callingFiles[i], callingLines[i]) } return } // Remove the lock and delete the entry from the map. if read { rwm.mu.RUnlock() } else { rwm.mu.Unlock() } delete(rwm.openLocks, id) } // RLock will read lock the RWMutex. The return value must be used as input // when calling RUnlock. func (rwm *RWMutex) RLock() int { return rwm.safeLock(true) } // RUnlock will read unlock the RWMutex. The return value of calling RLock must // be used as input. func (rwm *RWMutex) RUnlock(id int) { rwm.safeUnlock(true, id) } // Lock will lock the RWMutex. The return value must be used as input when // calling RUnlock. func (rwm *RWMutex) Lock() int { return rwm.safeLock(false) } // Unlock will unlock the RWMutex. The return value of calling Lock must be // used as input. func (rwm *RWMutex) Unlock(id int) { rwm.safeUnlock(false, id) } Sia-1.0.4/sync/lock_test.go000066400000000000000000000075371302033070500155270ustar00rootroot00000000000000package sync import ( "runtime" "sync" "testing" "time" ) // TestLowThreadLocking checks that locks are functional in the safelock // mechanism, only 2 threads are used to try and trigger a race condition. func TestLowThreadLocking(t *testing.T) { if testing.Short() { t.SkipNow() } // Create a value and lock a mutex to protect the value. value := 0 safeLock := New(time.Second, 1) outerID := safeLock.Lock() go func() { // Lock a mutex and read the value. Value should be 1, since the old // mutex was not released innerID := safeLock.Lock() defer safeLock.Unlock(innerID) if value != 1 { t.Fatal("Lock was grabbed incorrectly") } }() // After spawning the other thread, increment value. value = 1 safeLock.Unlock(outerID) } // TestHighThreadLocking tries to trigger race conditions while using lots of // threads and sleep tactics. func TestHighThreadLocking(t *testing.T) { if testing.Short() { t.SkipNow() } // Try to trigger a race condition by using lots of threads. for i := 0; i < 50; i++ { go func() { // Create a value and lock a mutex to protect the value. value := 0 safeLock := New(time.Second, 1) outerID := safeLock.Lock() go func() { // Lock a mutex and read the value. Value should be 1, since // the old mutex was not released innerID := safeLock.Lock() defer safeLock.Unlock(innerID) if value != 1 { t.Fatal("Lock was grabbed incorrectly") } }() // Some sleeps and a call to gosched to try and give the thread // control to the spawned thread. time.Sleep(time.Millisecond * 25) runtime.Gosched() time.Sleep(time.Millisecond * 25) value = 1 safeLock.Unlock(outerID) }() } } // TestReadLocking checks that the readlocks can overlap without interference // from a writelock. func TestReadLocking(t *testing.T) { if testing.Short() { t.SkipNow() } startTime := time.Now().Unix() value := 0 safeLock := New(time.Second, 1) writeID := safeLock.Lock() readThreads := 100 var wg sync.WaitGroup wg.Add(readThreads) for i := 0; i < readThreads; i++ { go func() { readID := safeLock.RLock() defer safeLock.RUnlock(readID) if value != 1 { t.Error("reading is not happening correctly") } // Sleep 250 milliseconds after grabbing the readlock. Because // there are a bunch of threads, if the readlocks are not grabbing // the lock in parallel the test will take a long time. time.Sleep(time.Millisecond * 250) wg.Done() }() } value = 1 // A combination of sleep and gosched to give priority to the other // threads. time.Sleep(time.Millisecond * 100) runtime.Gosched() time.Sleep(time.Millisecond * 100) safeLock.Unlock(writeID) // Wait for all of the threads to finish sleeping. wg.Wait() // Check that the whole test took under 3 seconds. If the readlocks were // efficiently being grabbed in parallel, the test should be subtantially // less than 3 seconds. if time.Now().Unix()-startTime > 3 { t.Error("test took too long to complete") } } // TestLockSafety checks that a safelock correctly unwinds a deadlock. func TestLockSafety(t *testing.T) { if testing.Short() { t.SkipNow() } startTime := time.Now().Unix() safeLock := New(time.Millisecond*25, 1) // Trigger a deadlock by writelocking twice. The deadlock detector should // resolve the issue. outerWrite := safeLock.Lock() innerWrite := safeLock.Lock() safeLock.Unlock(outerWrite) safeLock.Unlock(innerWrite) // Trigger a deadlock by readlocking and then writelocking. The deadlock // detector should resolve the issue. readID := safeLock.RLock() writeID := safeLock.Lock() safeLock.RUnlock(readID) safeLock.Unlock(writeID) // Check that the whole test took under 3 seconds. If the deadlock detector // is working, the time elapsed should be much less than 3 seconds. if time.Now().Unix()-startTime > 2 { t.Error("test took too long to complete") } } Sia-1.0.4/sync/threadgroup.go000066400000000000000000000105211302033070500160470ustar00rootroot00000000000000package sync import ( "errors" "sync" ) // ErrStopped is returned by ThreadGroup methods if Stop has already been // called. var ErrStopped = errors.New("ThreadGroup already stopped") // A ThreadGroup is a one-time-use object to manage the life cycle of a group // of threads. It is a sync.WaitGroup that provides functions for coordinating // actions and shutting down threads. After Stop() is called, the thread group // is no longer useful. // // It is safe to call Add(), Done(), and Stop() concurrently, however it is not // safe to nest calls to Add(). A simple example of a nested call to add would // be: // tg.Add() // tg.Add() // tg.Done() // tg.Done() type ThreadGroup struct { onStopFns []func() afterStopFns []func() once sync.Once stopChan chan struct{} bmu sync.Mutex // Ensures blocking between calls to 'Add', 'Flush', and 'Stop' mu sync.Mutex // Protects the 'onStopFns' and 'afterStopFns' variable wg sync.WaitGroup } // init creates the stop channel for the thread group. func (tg *ThreadGroup) init() { tg.stopChan = make(chan struct{}) } // isStopped will return true if Stop() has been called on the thread group. func (tg *ThreadGroup) isStopped() bool { tg.once.Do(tg.init) select { case <-tg.stopChan: return true default: return false } } // Add increments the thread group counter. func (tg *ThreadGroup) Add() error { tg.bmu.Lock() defer tg.bmu.Unlock() if tg.isStopped() { return ErrStopped } tg.wg.Add(1) return nil } // AfterStop ensures that a function will be called after Stop() has been // called and after all running routines have called Done(). The functions will // be called in reverse order to how they were added, similar to defer. If // Stop() has already been called, the input function will be called // immediately. // // The primary use of AfterStop is to allow code that opens and closes // resources to be positioned next to each other. The purpose is similar to // `defer`, except for resources that outlive the function which creates them. func (tg *ThreadGroup) AfterStop(fn func()) { tg.mu.Lock() defer tg.mu.Unlock() if tg.isStopped() { fn() return } tg.afterStopFns = append(tg.afterStopFns, fn) } // OnStop ensures that a function will be called after Stop() has been called, // and before blocking until all running routines have called Done(). It is // safe to use OnStop to coordinate the closing of long-running threads. The // OnStop functions will be called in the reverse order in which they were // added, similar to defer. If Stop() has already been called, the input // function will be called immediately. func (tg *ThreadGroup) OnStop(fn func()) { tg.mu.Lock() defer tg.mu.Unlock() if tg.isStopped() { fn() return } tg.onStopFns = append(tg.onStopFns, fn) } // Done decrements the thread group counter. func (tg *ThreadGroup) Done() { tg.wg.Done() } // Flush will block all calls to 'tg.Add' until all current routines have // called 'tg.Done'. This in effect 'flushes' the module, letting it complete // any tasks that are open before taking on new ones. func (tg *ThreadGroup) Flush() error { tg.bmu.Lock() defer tg.bmu.Unlock() if tg.isStopped() { return ErrStopped } tg.wg.Wait() return nil } // Stop will close the stop channel of the thread group, then call all 'OnStop' // functions in reverse order, then will wait until the thread group counter // reaches zero, then will call all of the 'AfterStop' functions in reverse // order. After Stop is called, most actions will return ErrStopped. func (tg *ThreadGroup) Stop() error { // Establish that Stop has been called. tg.bmu.Lock() defer tg.bmu.Unlock() if tg.isStopped() { return ErrStopped } close(tg.stopChan) tg.mu.Lock() for i := len(tg.onStopFns) - 1; i >= 0; i-- { tg.onStopFns[i]() } tg.onStopFns = nil tg.mu.Unlock() tg.wg.Wait() // After waiting for all resources to release the thread group, iterate // through the stop functions and call them in reverse oreder. tg.mu.Lock() for i := len(tg.afterStopFns) - 1; i >= 0; i-- { tg.afterStopFns[i]() } tg.afterStopFns = nil tg.mu.Unlock() return nil } // StopChan provides read-only access to the ThreadGroup's stopChan. Callers // should select on StopChan in order to interrupt long-running reads (such as // time.After). func (tg *ThreadGroup) StopChan() <-chan struct{} { tg.once.Do(tg.init) return tg.stopChan } Sia-1.0.4/sync/threadgroup_test.go000066400000000000000000000262611302033070500171160ustar00rootroot00000000000000package sync import ( "net" "os" "path/filepath" "sync" "testing" "time" "github.com/NebulousLabs/Sia/build" ) // TestThreadGroupStopEarly tests that a thread group can correctly interrupt // an ongoing process. func TestThreadGroupStopEarly(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() var tg ThreadGroup for i := 0; i < 10; i++ { err := tg.Add() if err != nil { t.Fatal(err) } go func() { defer tg.Done() select { case <-time.After(1 * time.Second): case <-tg.StopChan(): } }() } start := time.Now() err := tg.Stop() elapsed := time.Since(start) if err != nil { t.Fatal(err) } else if elapsed > 100*time.Millisecond { t.Fatal("Stop did not interrupt goroutines") } } // TestThreadGroupWait tests that a thread group will correctly wait for // existing processes to halt. func TestThreadGroupWait(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() var tg ThreadGroup for i := 0; i < 10; i++ { err := tg.Add() if err != nil { t.Fatal(err) } go func() { defer tg.Done() time.Sleep(time.Second) }() } start := time.Now() err := tg.Stop() elapsed := time.Since(start) if err != nil { t.Fatal(err) } else if elapsed < time.Millisecond*950 { t.Fatal("Stop did not wait for goroutines:", elapsed) } } // TestThreadGroupStop tests the behavior of a ThreadGroup after Stop has been // called. func TestThreadGroupStop(t *testing.T) { // Create a thread group and stop it. var tg ThreadGroup // Create an array to track the order of execution for OnStop and AfterStop // calls. var stopCalls []int // isStopped should return false if tg.isStopped() { t.Error("isStopped returns true on unstopped ThreadGroup") } // The cannel provided by StopChan should be open. select { case <-tg.StopChan(): t.Error("stop chan appears to be closed") default: } // OnStop and AfterStop should queue their functions, but not call them. // 'Add' and 'Done' are setup around the OnStop functions, to make sure // that the OnStop functions are called before waiting for all calls to // 'Done' to come through. // // Note: the practice of calling Add outside of OnStop and Done inside of // OnStop is a bad one - any call to tg.Flush() will cause a deadlock // because the stop functions will not be called but tg.Flush will be // waiting for the thread group counter to reach zero. err := tg.Add() if err != nil { t.Fatal(err) } err = tg.Add() if err != nil { t.Fatal(err) } tg.OnStop(func() { tg.Done() stopCalls = append(stopCalls, 1) }) tg.OnStop(func() { tg.Done() stopCalls = append(stopCalls, 2) }) tg.AfterStop(func() { stopCalls = append(stopCalls, 10) }) tg.AfterStop(func() { stopCalls = append(stopCalls, 20) }) // None of the stop calls should have been called yet. if len(stopCalls) != 0 { t.Fatal("Stop calls were called too early") } // Stop the thread group. err = tg.Stop() if err != nil { t.Fatal(err) } // isStopped should return true. if !tg.isStopped() { t.Error("isStopped returns false on stopped ThreadGroup") } // The cannel provided by StopChan should be closed. select { case <-tg.StopChan(): default: t.Error("stop chan appears to be closed") } // The OnStop calls should have been called first, in reverse order, and // the AfterStop calls should have been called second, in reverse order. if len(stopCalls) != 4 { t.Fatal("Stop did not call the stopping functions correctly") } if stopCalls[0] != 2 { t.Error("Stop called the stopping functions in the wrong order") } if stopCalls[1] != 1 { t.Error("Stop called the stopping functions in the wrong order") } if stopCalls[2] != 20 { t.Error("Stop called the stopping functions in the wrong order") } if stopCalls[3] != 10 { t.Error("Stop called the stopping functions in the wrong order") } // Add and Stop should return errors. err = tg.Add() if err != ErrStopped { t.Error("expected ErrStopped, got", err) } err = tg.Stop() if err != ErrStopped { t.Error("expected ErrStopped, got", err) } // OnStop and AfterStop should call their functions immediately now that // the thread group has stopped. onStopCalled := false tg.OnStop(func() { onStopCalled = true }) if !onStopCalled { t.Error("OnStop function not called immediately despite the thread group being closed already.") } afterStopCalled := false tg.AfterStop(func() { afterStopCalled = true }) if !afterStopCalled { t.Error("AfterStop function not called immediately despite the thread group being closed already.") } } // TestThreadGroupConcurrentAdd tests that Add can be called concurrently with Stop. func TestThreadGroupConcurrentAdd(t *testing.T) { if testing.Short() { t.SkipNow() } var tg ThreadGroup for i := 0; i < 10; i++ { go func() { err := tg.Add() if err != nil { return } defer tg.Done() select { case <-time.After(1 * time.Second): case <-tg.StopChan(): } }() } time.Sleep(10 * time.Millisecond) // wait for at least one Add err := tg.Stop() if err != nil { t.Fatal(err) } } // TestThreadGroupOnce tests that a zero-valued ThreadGroup's stopChan is // properly initialized. func TestThreadGroupOnce(t *testing.T) { tg := new(ThreadGroup) if tg.stopChan != nil { t.Error("expected nil stopChan") } // these methods should cause stopChan to be initialized tg.StopChan() if tg.stopChan == nil { t.Error("stopChan should have been initialized by StopChan") } tg = new(ThreadGroup) tg.isStopped() if tg.stopChan == nil { t.Error("stopChan should have been initialized by isStopped") } tg = new(ThreadGroup) tg.Add() if tg.stopChan == nil { t.Error("stopChan should have been initialized by Add") } tg = new(ThreadGroup) tg.Stop() if tg.stopChan == nil { t.Error("stopChan should have been initialized by Stop") } } // TestThreadGroupOnStop tests that Stop calls functions registered with // OnStop. func TestThreadGroupOnStop(t *testing.T) { if testing.Short() { t.SkipNow() } l, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatal(err) } // create ThreadGroup and register the closer var tg ThreadGroup tg.OnStop(func() { l.Close() }) // send on channel when listener is closed var closed bool tg.Add() go func() { defer tg.Done() _, err := l.Accept() closed = err != nil }() tg.Stop() if !closed { t.Fatal("Stop did not close listener") } } // TestThreadGroupRace tests that calling ThreadGroup methods concurrently // does not trigger the race detector. func TestThreadGroupRace(t *testing.T) { var tg ThreadGroup go tg.StopChan() go func() { if tg.Add() == nil { tg.Done() } }() err := tg.Stop() if err != nil { t.Fatal(err) } } // TestThreadGroupCloseAfterStop checks that an AfterStop function is // correctly called after the thread is stopped. func TestThreadGroupClosedAfterStop(t *testing.T) { var tg ThreadGroup var closed bool tg.AfterStop(func() { closed = true }) if closed { t.Fatal("close function should not have been called yet") } if err := tg.Stop(); err != nil { t.Fatal(err) } if !closed { t.Fatal("close function should have been called") } // Stop has already been called, so the close function should be called // immediately closed = false tg.AfterStop(func() { closed = true }) if !closed { t.Fatal("close function should have been called immediately") } } // TestThreadGroupSiaExample tries to use a thread group as it might be // expected to be used by a module of Sia. func TestThreadGroupSiaExample(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() testDir := build.TempDir("sync", "TestThreadGroupSiaExample") err := os.MkdirAll(testDir, 0700) if err != nil { t.Fatal(err) } var tg ThreadGroup // Open an example file. The file is expected to be used throughout the // lifetime of the module, and should not be closed until 'AfterStop' is // called. fileClosed := false file, err := os.Create(filepath.Join(testDir, "exampleFile.txt")) if err != nil { t.Fatal(err) } tg.AfterStop(func() { fileClosed = true err := file.Close() if err != nil { t.Fatal(err) } }) // Open a listener. The listener and handler thread should be closed before // the file is closed. listenerCleanedUp := false listener, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatal(err) } // Open a thread to accept calls from the listener. handlerFinishedChan := make(chan struct{}) go func() { for { _, err := listener.Accept() if err != nil { break } } handlerFinishedChan <- struct{}{} }() tg.OnStop(func() { err := listener.Close() if err != nil { t.Fatal(err) } <-handlerFinishedChan if fileClosed { t.Error("file should be open while the listener is shutting down") } listenerCleanedUp = true }) // Create a thread that does some stuff which takes time, and then closes. // Use Flush to clear out the process without closing the resources. threadFinished := false err = tg.Add() if err != nil { t.Fatal(err) } go func() { time.Sleep(time.Second) threadFinished = true tg.Done() }() tg.Flush() if !threadFinished { t.Error("call to Flush should have allowed the working thread to finish") } if listenerCleanedUp || fileClosed { t.Error("call to Flush resulted in permanent resources being closed") } // Create a thread that does some stuff which takes time, and then closes. // Use Stop to wait for the threead to finish and then check that all // resources have closed. threadFinished2 := false err = tg.Add() if err != nil { t.Fatal(err) } go func() { time.Sleep(time.Second) threadFinished2 = true tg.Done() }() tg.Stop() if !threadFinished || !listenerCleanedUp || !fileClosed { t.Error("stop did not block until all running resources had closed") } } // TestAddOnStop checks that you can safely call OnStop from under the // protection of an Add call. func TestAddOnStop(t *testing.T) { if testing.Short() { t.SkipNow() } t.Parallel() var tg ThreadGroup var data int addChan := make(chan struct{}) stopChan := make(chan struct{}) tg.OnStop(func() { close(stopChan) }) go func() { err := tg.Add() if err != nil { t.Fatal(err) } close(addChan) // Wait for the call to 'Stop' to be called in the parent thread, and // then queue a bunch of 'OnStop' and 'AfterStop' functions before // calling 'Done'. <-stopChan for i := 0; i < 10; i++ { tg.OnStop(func() { data++ }) tg.AfterStop(func() { data++ }) } tg.Done() }() // Wait for 'Add' to be called in the above thread, to guarantee that // OnStop and AfterStop will be called after 'Add' and 'Stop' have been // called together. <-addChan err := tg.Stop() if err != nil { t.Fatal(err) } if data != 20 { t.Error("20 calls were made to increment data, but value is", data) } } // BenchmarkThreadGroup times how long it takes to add a ton of threads and // trigger goroutines that call Done. func BenchmarkThreadGroup(b *testing.B) { var tg ThreadGroup for i := 0; i < b.N; i++ { tg.Add() go tg.Done() } tg.Stop() } // BenchmarkWaitGroup times how long it takes to add a ton of threads to a wait // group and trigger goroutines that call Done. func BenchmarkWaitGroup(b *testing.B) { var wg sync.WaitGroup for i := 0; i < b.N; i++ { wg.Add(1) go wg.Done() } wg.Wait() } Sia-1.0.4/sync/trymutex.go000066400000000000000000000024451302033070500154320ustar00rootroot00000000000000package sync import ( "sync" "time" ) // TryMutex provides a mutex that allows you to attempt to grab a mutex, and // then fail if the mutex is either not grabbed immediately or is not grabbed // by the specified duration. type TryMutex struct { once sync.Once lock chan struct{} } // init will create the channel that manages the lock. func (tm *TryMutex) init() { tm.lock = make(chan struct{}, 1) tm.lock <- struct{}{} } // Lock grabs a lock on the TryMutex, blocking until the lock is obtained. func (tm *TryMutex) Lock() { tm.once.Do(tm.init) <-tm.lock } // TryLock grabs a lock on the TryMutex, returning an error if the mutex is // already locked. func (tm *TryMutex) TryLock() bool { tm.once.Do(tm.init) select { case <-tm.lock: return true default: return false } } // TryLockTimed grabs a lock on the TryMutex, returning an error if the mutex // is not grabbed after the provided duration. func (tm *TryMutex) TryLockTimed(t time.Duration) bool { tm.once.Do(tm.init) select { case <-tm.lock: return true case <-time.After(t): return false } } // Unlock releases a lock on the TryMutex. func (tm *TryMutex) Unlock() { tm.once.Do(tm.init) select { case tm.lock <- struct{}{}: // Success - do nothing. default: panic("unlock called when TryMutex is not locked") } } Sia-1.0.4/sync/trymutex_test.go000066400000000000000000000075151302033070500164740ustar00rootroot00000000000000package sync import ( "sync" "testing" "time" ) // TestTryMutexBasicMutex verifies that Lock and Unlock work the same as a // normal mutex would. func TestTryMutexBasicMutex(t *testing.T) { // Check that two calls to lock will execute in the correct order. var tm TryMutex var data int tm.Lock() go func() { data = 15 tm.Unlock() }() tm.Lock() if data != 15 { t.Error("Locking did not safely protect the data") } tm.Unlock() } // TestTryMutexConcurrentLocking checks that doing lots of concurrent locks is // handled as expected. func TestTryMutexConcurrentLocking(t *testing.T) { if testing.Short() { t.SkipNow() } // Try executing multiple additions concurrently. var tm TryMutex var data int var wg sync.WaitGroup for i := 0; i < 250; i++ { wg.Add(1) go func() { tm.Lock() data++ tm.Unlock() wg.Done() }() } wg.Wait() if data != 250 { t.Error("Locking did not safely protect the data") } } // TestTryMutexBasicTryLock checks that a TryLock will succeed if nobody is // holding a lock, and will fail if the lock is being held. func TestTryMutexBasicTryLock(t *testing.T) { // Lock and then TryLock. var tm TryMutex tm.Lock() if tm.TryLock() { t.Error("TryLock should have failed") } tm.Unlock() tm.Lock() tm.Unlock() // TryLock and then TryLock. if !tm.TryLock() { t.Error("Could not get a blank lock") } if tm.TryLock() { t.Error("should not have been able to get the lock") } tm.Unlock() } // TestTryMutexConcurrentTries attempts to grab locks from many threads, giving // the race detector a chance to detect any issues. func TestTryMutexConncurrentTries(t *testing.T) { if testing.Short() { t.SkipNow() } // Try executing multiple additions concurrently. var tm TryMutex var data int var wg sync.WaitGroup for i := 0; i < 250; i++ { wg.Add(1) go func() { for !tm.TryLock() { } data++ tm.Unlock() wg.Done() }() } wg.Wait() if data != 250 { t.Error("Locking did not safely protect the data") } } // TestTryMutexTimed checks that a timed lock will correctly time out if it // cannot grab a lock. func TestTryMutexTimed(t *testing.T) { if testing.Short() { t.SkipNow() } var tm TryMutex tm.Lock() startTime := time.Now() if tm.TryLockTimed(time.Millisecond * 500) { t.Error("was able to grab a locked lock") } wait := time.Now().Sub(startTime) if wait < time.Millisecond*450 { t.Error("lock did not wait the correct amount of time before timing out", wait) } if wait > time.Millisecond*700 { t.Error("lock waited too long before timing out", wait) } tm.Unlock() if !tm.TryLockTimed(time.Millisecond * 1) { t.Error("Unable to get an unlocked lock") } tm.Unlock() } // TestTryMutexTimedConcurrent checks that a timed lock will correctly time out // if it cannot grab a lock. func TestTryMutexTimedConcurrent(t *testing.T) { if testing.Short() { t.SkipNow() } var tm TryMutex // Engage a lock and launch a gothread to wait for a lock, fail, and then // call unlock. tm.Lock() go func() { startTime := time.Now() if tm.TryLockTimed(time.Millisecond * 500) { t.Error("was able to grab a locked lock") } wait := time.Now().Sub(startTime) if wait < time.Millisecond*450 { t.Error("lock did not wait the correct amount of time before timing out:", wait) } if wait > time.Millisecond*700 { t.Error("lock waited too long before timing out", wait) } tm.Unlock() }() // Try to get a lock, but don't wait long enough. if tm.TryLockTimed(time.Millisecond * 250) { // Lock shoud time out because the gothread responsible for releasing // the lock will be idle for 500 milliseconds. t.Error("Lock should have timed out") } if !tm.TryLockTimed(time.Millisecond * 750) { // Lock should be successful - the above thread should finish in under // 750 milliseconds. t.Error("Lock should have been successful") } tm.Unlock() } Sia-1.0.4/types/000077500000000000000000000000001302033070500133655ustar00rootroot00000000000000Sia-1.0.4/types/block.go000066400000000000000000000125151302033070500150120ustar00rootroot00000000000000package types // block.go defines the Block type for Sia, and provides some helper functions // for working with blocks. import ( "encoding/json" "fmt" "io" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" ) const ( // BlockHeaderSize is the size, in bytes, of a block header. // 32 (ParentID) + 8 (Nonce) + 8 (Timestamp) + 32 (MerkleRoot) BlockHeaderSize = 80 ) type ( // A Block is a summary of changes to the state that have occurred since the // previous block. Blocks reference the ID of the previous block (their // "parent"), creating the linked-list commonly known as the blockchain. Their // primary function is to bundle together transactions on the network. Blocks // are created by "miners," who collect transactions from other nodes, and // then try to pick a Nonce that results in a block whose BlockID is below a // given Target. Block struct { ParentID BlockID `json:"parentid"` Nonce BlockNonce `json:"nonce"` Timestamp Timestamp `json:"timestamp"` MinerPayouts []SiacoinOutput `json:"minerpayouts"` Transactions []Transaction `json:"transactions"` } // A BlockHeader, when encoded, is an 80-byte constant size field // containing enough information to do headers-first block downloading. // Hashing the header results in the block ID. BlockHeader struct { ParentID BlockID `json:"parentid"` Nonce BlockNonce `json:"nonce"` Timestamp Timestamp `json:"timestamp"` MerkleRoot crypto.Hash `json:"merkleroot"` } BlockHeight uint64 BlockID crypto.Hash BlockNonce [8]byte ) // CalculateCoinbase calculates the coinbase for a given height. The coinbase // equation is: // // coinbase := max(InitialCoinbase - height, MinimumCoinbase) * SiacoinPrecision func CalculateCoinbase(height BlockHeight) Currency { base := InitialCoinbase - uint64(height) if uint64(height) > InitialCoinbase || base < MinimumCoinbase { base = MinimumCoinbase } return NewCurrency64(base).Mul(SiacoinPrecision) } // CalculateNumSiacoins calculates the number of siacoins in circulation at a // given height. func CalculateNumSiacoins(height BlockHeight) Currency { deflationBlocks := BlockHeight(InitialCoinbase - MinimumCoinbase) avgDeflationSiacoins := CalculateCoinbase(0).Add(CalculateCoinbase(height)).Div(NewCurrency64(2)) if height <= deflationBlocks { deflationSiacoins := avgDeflationSiacoins.Mul(NewCurrency64(uint64(height + 1))) return deflationSiacoins } deflationSiacoins := avgDeflationSiacoins.Mul(NewCurrency64(uint64(deflationBlocks + 1))) trailingSiacoins := NewCurrency64(uint64(height - deflationBlocks)).Mul(CalculateCoinbase(height)) return deflationSiacoins.Add(trailingSiacoins) } // ID returns the ID of a Block, which is calculated by hashing the header. func (h BlockHeader) ID() BlockID { return BlockID(crypto.HashObject(h)) } // CalculateSubsidy takes a block and a height and determines the block // subsidy. func (b Block) CalculateSubsidy(height BlockHeight) Currency { subsidy := CalculateCoinbase(height) for _, txn := range b.Transactions { for _, fee := range txn.MinerFees { subsidy = subsidy.Add(fee) } } return subsidy } // Header returns the header of a block. func (b Block) Header() BlockHeader { return BlockHeader{ ParentID: b.ParentID, Nonce: b.Nonce, Timestamp: b.Timestamp, MerkleRoot: b.MerkleRoot(), } } // ID returns the ID of a Block, which is calculated by hashing the // concatenation of the block's parent's ID, nonce, and the result of the // b.MerkleRoot(). It is equivalent to calling block.Header().ID() func (b Block) ID() BlockID { return b.Header().ID() } // MerkleRoot calculates the Merkle root of a Block. The leaves of the Merkle // tree are composed of the miner outputs (one leaf per payout), and the // transactions (one leaf per transaction). func (b Block) MerkleRoot() crypto.Hash { tree := crypto.NewTree() for _, payout := range b.MinerPayouts { tree.PushObject(payout) } for _, txn := range b.Transactions { tree.PushObject(txn) } return tree.Root() } // MinerPayoutID returns the ID of the miner payout at the given index, which // is calculated by hashing the concatenation of the BlockID and the payout // index. func (b Block) MinerPayoutID(i uint64) SiacoinOutputID { return SiacoinOutputID(crypto.HashAll( b.ID(), i, )) } // MarshalSia implements the encoding.SiaMarshaler interface. func (b Block) MarshalSia(w io.Writer) error { w.Write(b.ParentID[:]) w.Write(b.Nonce[:]) w.Write(encoding.EncUint64(uint64(b.Timestamp))) return encoding.NewEncoder(w).EncodeAll(b.MinerPayouts, b.Transactions) } // UnmarshalSia implements the encoding.SiaUnmarshaler interface. func (b *Block) UnmarshalSia(r io.Reader) error { io.ReadFull(r, b.ParentID[:]) io.ReadFull(r, b.Nonce[:]) tsBytes := make([]byte, 8) io.ReadFull(r, tsBytes) b.Timestamp = Timestamp(encoding.DecUint64(tsBytes)) return encoding.NewDecoder(r).DecodeAll(&b.MinerPayouts, &b.Transactions) } // MarshalJSON marshales a block id as a hex string. func (bid BlockID) MarshalJSON() ([]byte, error) { return json.Marshal(bid.String()) } // String prints the block id in hex. func (bid BlockID) String() string { return fmt.Sprintf("%x", bid[:]) } // UnmarshalJSON decodes the json hex string of the block id. func (bid *BlockID) UnmarshalJSON(b []byte) error { return (*crypto.Hash)(bid).UnmarshalJSON(b) } Sia-1.0.4/types/block_bench_test.go000066400000000000000000000013411302033070500172030ustar00rootroot00000000000000package types import ( "testing" "github.com/NebulousLabs/Sia/encoding" ) // BenchmarkEncodeEmptyBlock benchmarks encoding an empty block. // // i5-4670K, 9a90f86: 48 MB/s func BenchmarkEncodeBlock(b *testing.B) { var block Block b.SetBytes(int64(len(encoding.Marshal(block)))) for i := 0; i < b.N; i++ { encoding.Marshal(block) } } // BenchmarkDecodeEmptyBlock benchmarks decoding an empty block. // // i7-4770, b0b162d: 38 MB/s // i5-4670K, 9a90f86: 55 MB/s func BenchmarkDecodeEmptyBlock(b *testing.B) { var block Block encodedBlock := encoding.Marshal(block) b.SetBytes(int64(len(encodedBlock))) for i := 0; i < b.N; i++ { err := encoding.Unmarshal(encodedBlock, &block) if err != nil { b.Fatal(err) } } } Sia-1.0.4/types/block_test.go000066400000000000000000000171471302033070500160570ustar00rootroot00000000000000package types import ( "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" ) // TestCalculateCoinbase probes the CalculateCoinbase function. The test code // is probably too similar to the function code to be of value. func TestCalculateCoinbase(t *testing.T) { c := CalculateCoinbase(0) if c.Cmp(NewCurrency64(InitialCoinbase).Mul(SiacoinPrecision)) != 0 { t.Error("Unexpected CalculateCoinbase result") } c = CalculateCoinbase(1) if c.Cmp(NewCurrency64(InitialCoinbase-1).Mul(SiacoinPrecision)) != 0 { t.Error("Unexpected CalculateCoinbase result") } c = CalculateCoinbase(295000) if c.Cmp(NewCurrency64(MinimumCoinbase).Mul(SiacoinPrecision)) != 0 { t.Error(c) t.Error(NewCurrency64(MinimumCoinbase).Mul(SiacoinPrecision)) t.Error("Unexpected CalculateCoinbase result") } c = CalculateCoinbase(1000000000) if c.Cmp(NewCurrency64(MinimumCoinbase).Mul(SiacoinPrecision)) != 0 { t.Error(c) t.Error(NewCurrency64(MinimumCoinbase).Mul(SiacoinPrecision)) t.Error("Unexpected CalculateCoinbase result") } } // TestCalculateNumSiacoins checks that the siacoin calculator is correctly // determining the number of siacoins in circulation. The check is performed by // doing a naive computation, instead of by doing the optimized computation. func TestCalculateNumSiacoins(t *testing.T) { c := CalculateNumSiacoins(0) if c.Cmp(CalculateCoinbase(0)) != 0 { t.Error("unexpected circulation result for value 0, got", c) } if testing.Short() { t.SkipNow() } totalCoins := NewCurrency64(0) for i := BlockHeight(0); i < 500e3; i++ { totalCoins = totalCoins.Add(CalculateCoinbase(i)) if totalCoins.Cmp(CalculateNumSiacoins(i)) != 0 { t.Fatal("coin miscalculation", i, totalCoins, CalculateNumSiacoins(i)) } } } // TestBlockHeader checks that BlockHeader returns the correct value, and that // the hash is consistent with the old method for obtaining the hash. func TestBlockHeader(t *testing.T) { var b Block b.ParentID[1] = 1 b.Nonce[2] = 2 b.Timestamp = 3 b.MinerPayouts = []SiacoinOutput{{Value: NewCurrency64(4)}} b.Transactions = []Transaction{{ArbitraryData: [][]byte{{'5'}}}} id1 := b.ID() id2 := BlockID(crypto.HashBytes(encoding.Marshal(b.Header()))) id3 := BlockID(crypto.HashAll( b.ParentID, b.Nonce, b.Timestamp, b.MerkleRoot(), )) if id1 != id2 || id2 != id3 || id3 != id1 { t.Error("Methods for getting block id don't return the same results") } } // TestBlockID probes the ID function of the block type. func TestBlockID(t *testing.T) { // Create a bunch of different blocks and check that all of them have // unique ids. var b Block var ids []BlockID ids = append(ids, b.ID()) b.ParentID[0] = 1 ids = append(ids, b.ID()) b.Nonce[0] = 45 ids = append(ids, b.ID()) b.Timestamp = CurrentTimestamp() ids = append(ids, b.ID()) b.MinerPayouts = append(b.MinerPayouts, SiacoinOutput{Value: CalculateCoinbase(0)}) ids = append(ids, b.ID()) b.MinerPayouts = append(b.MinerPayouts, SiacoinOutput{Value: CalculateCoinbase(0)}) ids = append(ids, b.ID()) b.Transactions = append(b.Transactions, Transaction{MinerFees: []Currency{CalculateCoinbase(1)}}) ids = append(ids, b.ID()) b.Transactions = append(b.Transactions, Transaction{MinerFees: []Currency{CalculateCoinbase(1)}}) ids = append(ids, b.ID()) knownIDs := make(map[BlockID]struct{}) for i, id := range ids { _, exists := knownIDs[id] if exists { t.Error("id repeat for index", i) } knownIDs[id] = struct{}{} } } // TestHeaderID probes the ID function of the BlockHeader type. func TestHeaderID(t *testing.T) { // Create a bunch of different blocks and check that all of them have // unique ids. var blocks []Block var b Block blocks = append(blocks, b) b.ParentID[0] = 1 blocks = append(blocks, b) b.Nonce[0] = 45 blocks = append(blocks, b) b.Timestamp = CurrentTimestamp() blocks = append(blocks, b) b.MinerPayouts = append(b.MinerPayouts, SiacoinOutput{Value: CalculateCoinbase(0)}) blocks = append(blocks, b) b.MinerPayouts = append(b.MinerPayouts, SiacoinOutput{Value: CalculateCoinbase(0)}) blocks = append(blocks, b) b.Transactions = append(b.Transactions, Transaction{MinerFees: []Currency{CalculateCoinbase(1)}}) blocks = append(blocks, b) b.Transactions = append(b.Transactions, Transaction{MinerFees: []Currency{CalculateCoinbase(1)}}) blocks = append(blocks, b) knownIDs := make(map[BlockID]struct{}) for i, block := range blocks { blockID := block.ID() headerID := block.Header().ID() if blockID != headerID { t.Error("headerID does not match blockID for index", i) } _, exists := knownIDs[headerID] if exists { t.Error("id repeat for index", i) } knownIDs[headerID] = struct{}{} } } // TestBlockCalculateSubsidy probes the CalculateSubsidy function of the block // type. func TestBlockCalculateSubsidy(t *testing.T) { // All tests are done at height = 0. coinbase := CalculateCoinbase(0) // Calculate the subsidy on a block with 0 fees at height 0. Result should // be 300,000. var b Block if b.CalculateSubsidy(0).Cmp(coinbase) != 0 { t.Error("subsidy is miscalculated for an empty block") } // Calculate when there is a fee in a transcation. expected := coinbase.Add(NewCurrency64(123)) txn := Transaction{ MinerFees: []Currency{NewCurrency64(123)}, } b.Transactions = append(b.Transactions, txn) if b.CalculateSubsidy(0).Cmp(expected) != 0 { t.Error("subsidy is miscalculated for a block with a single transaction") } // Add a single no-fee transaction and check again. txn = Transaction{ ArbitraryData: [][]byte{{'6'}}, } b.Transactions = append(b.Transactions, txn) if b.CalculateSubsidy(0).Cmp(expected) != 0 { t.Error("subsidy is miscalculated with empty transactions.") } // Add a transaction with multiple fees. expected = expected.Add(NewCurrency64(1 + 2 + 3)) txn = Transaction{ MinerFees: []Currency{ NewCurrency64(1), NewCurrency64(2), NewCurrency64(3), }, } b.Transactions = append(b.Transactions, txn) if b.CalculateSubsidy(0).Cmp(expected) != 0 { t.Error("subsidy is miscalculated for a block with a single transaction") } // Add an empty transaction to the beginning. txn = Transaction{ ArbitraryData: [][]byte{{'7'}}, } b.Transactions = append([]Transaction{txn}, b.Transactions...) if b.CalculateSubsidy(0).Cmp(expected) != 0 { t.Error("subsidy is miscalculated with empty transactions.") } } // TestBlockMinerPayoutID probes the MinerPayout function of the block type. func TestBlockMinerPayoutID(t *testing.T) { // Create a block with 2 miner payouts, and check that each payout has a // different id, and that the id is dependent on the block id. var ids []SiacoinOutputID b := Block{ MinerPayouts: []SiacoinOutput{ {Value: CalculateCoinbase(0)}, {Value: CalculateCoinbase(0)}, }, } ids = append(ids, b.MinerPayoutID(1), b.MinerPayoutID(2)) b.ParentID[0] = 1 ids = append(ids, b.MinerPayoutID(1), b.MinerPayoutID(2)) knownIDs := make(map[SiacoinOutputID]struct{}) for i, id := range ids { _, exists := knownIDs[id] if exists { t.Error("id repeat for index", i) } knownIDs[id] = struct{}{} } } // TestBlockEncodes probes the MarshalSia and UnmarshalSia methods of the // Block type. func TestBlockEncoding(t *testing.T) { b := Block{ MinerPayouts: []SiacoinOutput{ {Value: CalculateCoinbase(0)}, {Value: CalculateCoinbase(0)}, }, } var decB Block err := encoding.Unmarshal(encoding.Marshal(b), &decB) if err != nil { t.Fatal(err) } if len(decB.MinerPayouts) != len(b.MinerPayouts) || decB.MinerPayouts[0].Value.Cmp(b.MinerPayouts[0].Value) != 0 || decB.MinerPayouts[1].Value.Cmp(b.MinerPayouts[1].Value) != 0 { t.Fatal("block changed after encode/decode:", b, decB) } } Sia-1.0.4/types/constants.go000066400000000000000000000443231302033070500157360ustar00rootroot00000000000000package types // constants.go contains the Sia constants. Depending on which build tags are // used, the constants will be initialized to different values. // // CONTRIBUTE: We don't have way to check that the non-test constants are all // sane, plus we have no coverage for them. import ( "math/big" "github.com/NebulousLabs/Sia/build" ) var ( BlockSizeLimit = uint64(2e6) RootDepth = Target{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255} BlockFrequency BlockHeight MaturityDelay BlockHeight GenesisTimestamp Timestamp RootTarget Target MedianTimestampWindow = uint64(11) TargetWindow BlockHeight MaxAdjustmentUp *big.Rat MaxAdjustmentDown *big.Rat FutureThreshold Timestamp ExtremeFutureThreshold Timestamp SiafundCount = NewCurrency64(10000) SiafundPortion = big.NewRat(39, 1000) SiacoinPrecision = NewCurrency(new(big.Int).Exp(big.NewInt(10), big.NewInt(24), nil)) InitialCoinbase = uint64(300e3) MinimumCoinbase uint64 GenesisSiafundAllocation []SiafundOutput GenesisBlock Block // The GenesisID is used in many places. Calculating it once saves lots of // redundant computation. GenesisID BlockID ) // init checks which build constant is in place and initializes the variables // accordingly. func init() { if build.Release == "dev" { // 'dev' settings are for small developer testnets, usually on the same // computer. Settings are slow enough that a small team of developers // can coordinate their actions over a the developer testnets, but fast // enough that there isn't much time wasted on waiting for things to // happen. BlockFrequency = 12 // 12 seconds: slow enough for developers to see ~each block, fast enough that blocks don't waste time. MaturityDelay = 10 // 60 seconds before a delayed output matures. GenesisTimestamp = Timestamp(1424139000) // Change as necessary. RootTarget = Target{0, 0, 10} // Standard developer CPUs will be able to mine blocks with the race library activated. TargetWindow = 20 // Difficulty is adjusted based on prior 20 blocks. MaxAdjustmentUp = big.NewRat(120, 100) // Difficulty adjusts quickly. MaxAdjustmentDown = big.NewRat(100, 120) // Difficulty adjusts quickly. FutureThreshold = 2 * 60 // 2 minutes. ExtremeFutureThreshold = 4 * 60 // 4 minutes. MinimumCoinbase = 30e3 GenesisSiafundAllocation = []SiafundOutput{ { Value: NewCurrency64(2000), UnlockHash: UnlockHash{214, 166, 197, 164, 29, 201, 53, 236, 106, 239, 10, 158, 127, 131, 20, 138, 63, 221, 230, 16, 98, 247, 32, 77, 210, 68, 116, 12, 241, 89, 27, 223}, }, { Value: NewCurrency64(7000), UnlockHash: UnlockHash{209, 246, 228, 60, 248, 78, 242, 110, 9, 8, 227, 248, 225, 216, 163, 52, 142, 93, 47, 176, 103, 41, 137, 80, 212, 8, 132, 58, 241, 189, 2, 17}, }, { Value: NewCurrency64(1000), UnlockHash: UnlockConditions{}.UnlockHash(), }, } } else if build.Release == "testing" { // 'testing' settings are for automatic testing, and create much faster // environments than a human can interact with. BlockFrequency = 1 // As fast as possible MaturityDelay = 3 GenesisTimestamp = CurrentTimestamp() - 1e6 RootTarget = Target{128} // Takes an expected 2 hashes; very fast for testing but still probes 'bad hash' code. // A restrictive difficulty clamp prevents the difficulty from climbing // during testing, as the resolution on the difficulty adjustment is // only 1 second and testing mining should be happening substantially // faster than that. TargetWindow = 200 MaxAdjustmentUp = big.NewRat(10001, 10000) MaxAdjustmentDown = big.NewRat(9999, 10000) FutureThreshold = 3 // 3 seconds ExtremeFutureThreshold = 6 // 6 seconds MinimumCoinbase = 299990 // Minimum coinbase is hit after 10 blocks to make testing minimum-coinbase code easier. GenesisSiafundAllocation = []SiafundOutput{ { Value: NewCurrency64(2000), UnlockHash: UnlockHash{214, 166, 197, 164, 29, 201, 53, 236, 106, 239, 10, 158, 127, 131, 20, 138, 63, 221, 230, 16, 98, 247, 32, 77, 210, 68, 116, 12, 241, 89, 27, 223}, }, { Value: NewCurrency64(7000), UnlockHash: UnlockHash{209, 246, 228, 60, 248, 78, 242, 110, 9, 8, 227, 248, 225, 216, 163, 52, 142, 93, 47, 176, 103, 41, 137, 80, 212, 8, 132, 58, 241, 189, 2, 17}, }, { Value: NewCurrency64(1000), UnlockHash: UnlockConditions{}.UnlockHash(), }, } } else if build.Release == "standard" { // 'standard' settings are for the full network. They are slow enough // that the network is secure in a real-world byzantine environment. // A block time of 1 block per 10 minutes is chosen to follow Bitcoin's // example. The security lost by lowering the block time is not // insignificant, and the convenience gained by lowering the blocktime // even down to 90 seconds is not significant. I do feel that 10 // minutes could even be too short, but it has worked well for Bitcoin. BlockFrequency = 600 // Payouts take 1 day to mature. This is to prevent a class of double // spending attacks parties unintentionally spend coins that will stop // existing after a blockchain reorganization. There are multiple // classes of payouts in Sia that depend on a previous block - if that // block changes, then the output changes and the previously existing // output ceases to exist. This delay stops both unintentional double // spending and stops a small set of long-range mining attacks. MaturityDelay = 144 // The genesis timestamp is set to June 6th, because that is when the // 100-block developer premine started. The trailing zeroes are a // bonus, and make the timestamp easier to memorize. GenesisTimestamp = Timestamp(1433600000) // June 6th, 2015 @ 2:13pm UTC. // The RootTarget was set such that the developers could reasonable // premine 100 blocks in a day. It was known to the developrs at launch // this this was at least one and perhaps two orders of magnitude too // small. RootTarget = Target{0, 0, 0, 0, 32} // When the difficulty is adjusted, it is adjusted by looking at the // timestamp of the 1000th previous block. This minimizes the abilities // of miners to attack the network using rogue timestamps. TargetWindow = 1e3 // The difficutly adjustment is clamped to 2.5x every 500 blocks. This // corresponds to 6.25x every 2 weeks, which can be compared to // Bitcoin's clamp of 4x every 2 weeks. The difficulty clamp is // primarily to stop difficulty raising attacks. Sia's safety margin is // similar to Bitcoin's despite the looser clamp because Sia's // difficulty is adjusted four times as often. This does result in // greater difficulty oscillation, a tradeoff that was chosen to be // acceptable due to Sia's more vulnerable position as an altcoin. MaxAdjustmentUp = big.NewRat(25, 10) MaxAdjustmentDown = big.NewRat(10, 25) // Blocks will not be accepted if their timestamp is more than 3 hours // into the future, but will be accepted as soon as they are no longer // 3 hours into the future. Blocks that are greater than 5 hours into // the future are rejected outright, as it is assumed that by the time // 2 hours have passed, those blocks will no longer be on the longest // chain. Blocks cannot be kept forever because this opens a DoS // vector. FutureThreshold = 3 * 60 * 60 // 3 hours. ExtremeFutureThreshold = 5 * 60 * 60 // 5 hours. // The minimum coinbase is set to 30,000. Because the coinbase // decreases by 1 every time, it means that Sia's coinbase will have an // increasingly potent dropoff for about 5 years, until inflation more // or less permanently settles around 2%. MinimumCoinbase = 30e3 GenesisSiafundAllocation = []SiafundOutput{ { Value: NewCurrency64(2), UnlockHash: UnlockHash{4, 57, 229, 188, 127, 20, 204, 245, 211, 167, 232, 130, 208, 64, 146, 62, 69, 98, 81, 102, 221, 7, 123, 100, 70, 107, 199, 113, 121, 26, 198, 252}, }, { Value: NewCurrency64(6), UnlockHash: UnlockHash{4, 158, 29, 42, 105, 119, 43, 5, 138, 72, 190, 190, 101, 114, 79, 243, 189, 248, 208, 151, 30, 187, 233, 148, 225, 233, 28, 159, 19, 232, 75, 244}, }, { Value: NewCurrency64(7), UnlockHash: UnlockHash{8, 7, 66, 250, 25, 74, 247, 108, 162, 79, 220, 151, 202, 228, 241, 11, 130, 138, 13, 248, 193, 167, 136, 197, 65, 63, 234, 174, 205, 216, 71, 230}, }, { Value: NewCurrency64(8), UnlockHash: UnlockHash{44, 106, 239, 51, 138, 102, 242, 19, 204, 197, 248, 178, 219, 122, 152, 251, 19, 20, 52, 32, 175, 32, 4, 156, 73, 33, 163, 165, 222, 184, 217, 218}, }, { Value: NewCurrency64(3), UnlockHash: UnlockHash{44, 163, 31, 233, 74, 103, 55, 132, 230, 159, 97, 78, 149, 147, 65, 110, 164, 211, 105, 173, 158, 29, 202, 43, 85, 217, 85, 75, 83, 37, 205, 223}, }, { Value: NewCurrency64(1), UnlockHash: UnlockHash{51, 151, 146, 84, 199, 7, 59, 89, 111, 172, 227, 200, 62, 55, 165, 253, 238, 186, 28, 145, 47, 137, 200, 15, 70, 199, 187, 125, 243, 104, 179, 240}, }, { Value: NewCurrency64(10), UnlockHash: UnlockHash{53, 118, 253, 229, 254, 229, 28, 131, 233, 156, 108, 58, 197, 152, 17, 160, 74, 252, 11, 49, 112, 240, 66, 119, 40, 98, 114, 251, 5, 86, 233, 117}, }, { Value: NewCurrency64(50), UnlockHash: UnlockHash{56, 219, 3, 50, 28, 3, 166, 95, 141, 163, 202, 35, 60, 199, 219, 10, 151, 176, 228, 97, 176, 133, 189, 33, 211, 202, 83, 197, 31, 208, 254, 193}, }, { Value: NewCurrency64(75), UnlockHash: UnlockHash{68, 190, 140, 87, 96, 232, 150, 32, 161, 177, 204, 65, 228, 223, 87, 217, 134, 90, 25, 56, 51, 45, 72, 107, 129, 12, 29, 202, 6, 7, 50, 13}, }, { Value: NewCurrency64(10), UnlockHash: UnlockHash{69, 14, 201, 200, 90, 73, 245, 45, 154, 94, 161, 19, 199, 241, 203, 56, 13, 63, 5, 220, 121, 245, 247, 52, 194, 181, 252, 76, 130, 6, 114, 36}, }, { Value: NewCurrency64(10), UnlockHash: UnlockHash{72, 128, 253, 207, 169, 48, 1, 26, 237, 205, 169, 102, 196, 224, 42, 186, 95, 151, 59, 226, 203, 136, 251, 223, 165, 38, 88, 110, 47, 213, 121, 224}, }, { Value: NewCurrency64(50), UnlockHash: UnlockHash{72, 130, 164, 227, 218, 28, 60, 15, 56, 151, 212, 242, 77, 131, 232, 131, 42, 57, 132, 173, 113, 118, 66, 183, 38, 79, 96, 178, 105, 108, 26, 247}, }, { Value: NewCurrency64(10), UnlockHash: UnlockHash{74, 210, 58, 228, 111, 69, 253, 120, 53, 195, 110, 26, 115, 76, 211, 202, 199, 159, 204, 14, 78, 92, 14, 131, 250, 22, 141, 236, 154, 44, 39, 135}, }, { Value: NewCurrency64(15), UnlockHash: UnlockHash{85, 198, 154, 41, 196, 116, 226, 114, 202, 94, 214, 147, 87, 84, 247, 164, 195, 79, 58, 123, 26, 33, 68, 65, 116, 79, 181, 241, 241, 208, 215, 184}, }, { Value: NewCurrency64(121), UnlockHash: UnlockHash{87, 239, 83, 125, 152, 14, 19, 22, 203, 136, 46, 192, 203, 87, 224, 190, 77, 236, 125, 18, 142, 223, 146, 70, 16, 23, 252, 19, 100, 69, 91, 111}, }, { Value: NewCurrency64(222), UnlockHash: UnlockHash{91, 201, 101, 11, 188, 40, 35, 111, 236, 133, 31, 124, 97, 246, 140, 136, 143, 245, 152, 174, 111, 245, 188, 124, 21, 125, 187, 192, 203, 92, 253, 57}, }, { Value: NewCurrency64(10), UnlockHash: UnlockHash{110, 240, 238, 173, 78, 138, 185, 138, 179, 227, 135, 153, 54, 132, 46, 62, 226, 206, 204, 35, 174, 107, 156, 15, 142, 2, 93, 132, 163, 60, 50, 89}, }, { Value: NewCurrency64(3), UnlockHash: UnlockHash{114, 58, 147, 44, 64, 69, 72, 184, 65, 178, 213, 94, 157, 44, 88, 106, 92, 31, 145, 193, 215, 200, 215, 233, 99, 116, 36, 197, 160, 70, 79, 153}, }, { Value: NewCurrency64(1), UnlockHash: UnlockHash{123, 106, 229, 101, 220, 252, 50, 203, 38, 183, 133, 152, 250, 167, 210, 155, 252, 102, 150, 29, 187, 3, 178, 53, 11, 145, 143, 33, 166, 115, 250, 40}, }, { Value: NewCurrency64(5), UnlockHash: UnlockHash{124, 101, 207, 175, 50, 119, 207, 26, 62, 15, 247, 141, 150, 174, 73, 247, 238, 28, 77, 255, 222, 104, 166, 244, 112, 86, 227, 80, 215, 45, 69, 143}, }, { Value: NewCurrency64(10), UnlockHash: UnlockHash{130, 184, 72, 15, 227, 79, 217, 205, 120, 254, 67, 69, 10, 49, 76, 194, 222, 30, 242, 62, 88, 179, 51, 117, 27, 166, 140, 6, 7, 22, 222, 185}, }, { Value: NewCurrency64(25), UnlockHash: UnlockHash{134, 137, 198, 172, 96, 54, 45, 10, 100, 128, 91, 225, 226, 134, 143, 108, 31, 70, 187, 228, 54, 212, 70, 229, 149, 57, 64, 166, 153, 123, 238, 180}, }, { Value: NewCurrency64(1), UnlockHash: UnlockHash{143, 253, 118, 229, 109, 181, 141, 224, 91, 144, 123, 160, 203, 221, 119, 104, 172, 13, 105, 77, 171, 185, 122, 54, 229, 168, 6, 130, 160, 130, 182, 151}, }, { Value: NewCurrency64(8), UnlockHash: UnlockHash{147, 108, 249, 16, 36, 249, 108, 184, 196, 212, 241, 120, 219, 63, 45, 184, 86, 53, 96, 207, 130, 96, 210, 251, 136, 9, 193, 160, 131, 198, 221, 185}, }, { Value: NewCurrency64(58), UnlockHash: UnlockHash{155, 79, 89, 28, 69, 71, 239, 198, 246, 2, 198, 254, 92, 59, 192, 205, 229, 152, 36, 186, 110, 122, 233, 221, 76, 143, 3, 238, 89, 231, 192, 23}, }, { Value: NewCurrency64(2), UnlockHash: UnlockHash{156, 32, 76, 105, 213, 46, 66, 50, 27, 85, 56, 9, 106, 193, 80, 145, 19, 101, 84, 177, 145, 4, 125, 28, 79, 252, 43, 83, 118, 110, 206, 247}, }, { Value: NewCurrency64(23), UnlockHash: UnlockHash{157, 169, 134, 24, 254, 22, 58, 188, 119, 87, 201, 238, 55, 168, 194, 131, 88, 18, 39, 168, 37, 2, 198, 194, 93, 202, 116, 146, 189, 17, 108, 44}, }, { Value: NewCurrency64(10), UnlockHash: UnlockHash{158, 51, 104, 36, 242, 114, 67, 16, 168, 230, 4, 111, 241, 72, 5, 14, 182, 102, 169, 156, 144, 220, 103, 117, 223, 8, 58, 187, 124, 102, 80, 44}, }, { Value: NewCurrency64(1), UnlockHash: UnlockHash{160, 175, 59, 33, 223, 30, 82, 60, 34, 110, 28, 203, 249, 93, 3, 16, 218, 12, 250, 206, 138, 231, 85, 67, 69, 191, 68, 198, 160, 87, 154, 68}, }, { Value: NewCurrency64(75), UnlockHash: UnlockHash{163, 94, 51, 220, 14, 144, 83, 112, 62, 10, 0, 173, 161, 234, 211, 176, 186, 84, 9, 189, 250, 111, 33, 231, 114, 87, 100, 75, 72, 217, 11, 26}, }, { Value: NewCurrency64(3), UnlockHash: UnlockHash{170, 7, 138, 116, 205, 20, 132, 197, 166, 251, 75, 93, 69, 6, 109, 244, 212, 119, 173, 114, 34, 18, 25, 21, 111, 203, 203, 253, 138, 104, 27, 36}, }, { Value: NewCurrency64(90), UnlockHash: UnlockHash{173, 120, 128, 104, 186, 86, 151, 140, 191, 23, 231, 193, 77, 245, 243, 104, 196, 55, 155, 243, 111, 15, 84, 139, 148, 187, 173, 47, 104, 69, 141, 39}, }, { Value: NewCurrency64(20), UnlockHash: UnlockHash{179, 185, 228, 166, 139, 94, 13, 193, 255, 227, 174, 99, 120, 105, 109, 221, 247, 4, 155, 243, 229, 37, 26, 98, 222, 12, 91, 80, 223, 33, 61, 56}, }, { Value: NewCurrency64(5), UnlockHash: UnlockHash{193, 49, 103, 20, 170, 135, 182, 85, 149, 18, 159, 194, 152, 120, 162, 208, 49, 158, 220, 188, 114, 79, 1, 131, 62, 27, 86, 57, 244, 46, 64, 66}, }, { Value: NewCurrency64(1), UnlockHash: UnlockHash{196, 71, 45, 222, 0, 21, 12, 121, 197, 224, 101, 65, 40, 57, 19, 119, 112, 205, 166, 23, 2, 91, 75, 231, 69, 143, 221, 68, 245, 75, 7, 52}, }, { Value: NewCurrency64(44), UnlockHash: UnlockHash{196, 214, 236, 211, 227, 216, 152, 127, 164, 2, 235, 14, 235, 46, 142, 231, 83, 38, 7, 131, 208, 29, 179, 189, 62, 88, 129, 180, 119, 158, 214, 97}, }, { Value: NewCurrency64(23), UnlockHash: UnlockHash{206, 58, 114, 148, 131, 49, 87, 197, 86, 18, 216, 26, 62, 79, 152, 175, 33, 4, 132, 160, 108, 231, 53, 200, 48, 76, 125, 94, 156, 85, 32, 130}, }, { Value: NewCurrency64(80), UnlockHash: UnlockHash{200, 103, 135, 126, 197, 2, 203, 63, 241, 6, 245, 195, 220, 102, 27, 74, 232, 249, 201, 86, 207, 34, 51, 26, 180, 151, 136, 108, 112, 56, 132, 72}, }, { Value: NewCurrency64(2), UnlockHash: UnlockHash{200, 249, 245, 218, 58, 253, 76, 250, 88, 114, 70, 239, 14, 2, 250, 123, 10, 192, 198, 61, 187, 155, 247, 152, 165, 174, 198, 24, 142, 39, 177, 119}, }, { Value: NewCurrency64(1), UnlockHash: UnlockHash{209, 1, 199, 184, 186, 57, 21, 137, 33, 252, 219, 184, 130, 38, 32, 98, 63, 252, 250, 79, 70, 146, 169, 78, 180, 161, 29, 93, 38, 45, 175, 176}, }, { Value: NewCurrency64(2), UnlockHash: UnlockHash{212, 107, 233, 43, 185, 138, 79, 253, 12, 237, 214, 17, 219, 198, 151, 92, 81, 129, 17, 120, 139, 58, 66, 119, 126, 220, 132, 136, 3, 108, 57, 58}, }, { Value: NewCurrency64(3), UnlockHash: UnlockHash{214, 244, 146, 173, 173, 80, 33, 185, 29, 133, 77, 167, 185, 1, 38, 23, 111, 179, 104, 150, 105, 162, 120, 26, 245, 63, 114, 119, 52, 1, 44, 222}, }, { Value: NewCurrency64(1), UnlockHash: UnlockHash{217, 218, 172, 16, 53, 134, 160, 226, 44, 138, 93, 53, 181, 62, 4, 209, 190, 27, 0, 93, 105, 17, 169, 61, 98, 145, 131, 112, 121, 55, 97, 184}, }, { Value: NewCurrency64(1), UnlockHash: UnlockHash{223, 162, 172, 55, 54, 193, 37, 142, 200, 213, 230, 48, 186, 145, 184, 206, 15, 225, 167, 19, 37, 70, 38, 48, 135, 87, 205, 81, 187, 237, 181, 180}, }, { Value: NewCurrency64(1), UnlockHash: UnlockHash{241, 46, 139, 41, 40, 63, 47, 169, 131, 173, 124, 246, 228, 213, 102, 44, 100, 217, 62, 237, 133, 154, 248, 69, 228, 2, 36, 206, 47, 250, 249, 170}, }, { Value: NewCurrency64(50), UnlockHash: UnlockHash{241, 50, 229, 211, 66, 32, 115, 241, 117, 87, 180, 239, 76, 246, 14, 129, 105, 181, 153, 105, 105, 203, 229, 237, 23, 130, 193, 170, 100, 201, 38, 71}, }, { Value: NewCurrency64(8841), UnlockHash: UnlockHash{125, 12, 68, 247, 102, 78, 45, 52, 229, 62, 253, 224, 102, 26, 111, 98, 142, 201, 38, 71, 133, 174, 142, 60, 215, 201, 115, 232, 209, 144, 195, 201}, }, } } // Create the genesis block. GenesisBlock = Block{ Timestamp: GenesisTimestamp, Transactions: []Transaction{ {SiafundOutputs: GenesisSiafundAllocation}, }, } // Calculate the genesis ID. GenesisID = GenesisBlock.ID() } Sia-1.0.4/types/constants_test.go000066400000000000000000000011311302033070500167630ustar00rootroot00000000000000package types import ( "testing" "github.com/NebulousLabs/Sia/build" ) // TestCheckBuildConstants checks that the required build constants have been // set. func TestCheckBuildConstants(t *testing.T) { // Verify that the build has been set to 'testing'. if build.Release != "testing" { t.Error("build.Release needs to be set to \"testing\"") t.Error(build.Release) } if testing.Short() { t.SkipNow() } // Verify that, for the longer tests, the 'debug' build tag has been used. if !build.DEBUG { t.Error("DEBUG needs to be enabled for testing to work.") t.Error(build.DEBUG) } } Sia-1.0.4/types/currency.go000066400000000000000000000161701302033070500155530ustar00rootroot00000000000000package types // currency.go defines the internal currency object. One design goal of the // currency type is immutability: the currency type should be safe to pass // directly to other objects and packages. The currency object should never // have a negative value. The currency should never overflow. There is a // maximum size value that can be encoded (around 10^10^20), however exceeding // this value will not result in overflow. import ( "bytes" "errors" "fmt" "io" "math" "math/big" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/encoding" ) type ( // A Currency represents a number of siacoins or siafunds. Internally, a // Currency value is unbounded; however, Currency values sent over the wire // protocol are subject to a maximum size of 255 bytes (approximately 10^614). // Unlike the math/big library, whose methods modify their receiver, all // arithmetic Currency methods return a new value. Currency cannot be negative. Currency struct { i big.Int } ) var ( // ZeroCurrency defines a currency of value zero. ZeroCurrency = NewCurrency64(0) // ErrNegativeCurrency is the error that is returned if performing an // operation results in a negative currency. ErrNegativeCurrency = errors.New("negative currency not allowed") // ErrUint64Overflow is the error that is returned if converting to a // unit64 would cause an overflow. ErrUint64Overflow = errors.New("cannot return the uint64 of this currency - result is an overflow") ) // NewCurrency creates a Currency value from a big.Int. Undefined behavior // occurs if a negative input is used. func NewCurrency(b *big.Int) (c Currency) { if b.Sign() < 0 { build.Critical(ErrNegativeCurrency) } else { c.i = *b } return } // NewCurrency64 creates a Currency value from a uint64. func NewCurrency64(x uint64) (c Currency) { c.i.SetUint64(x) return } // Add returns a new Currency value c = x + y func (x Currency) Add(y Currency) (c Currency) { c.i.Add(&x.i, &y.i) return } // Big returns the value of c as a *big.Int. Importantly, it does not provide // access to the c's internal big.Int object, only a copy. func (c Currency) Big() *big.Int { return new(big.Int).Set(&c.i) } // Cmp compares two Currency values. The return value follows the convention // of math/big. func (x Currency) Cmp(y Currency) int { return x.i.Cmp(&y.i) } // Cmp64 compares x to a uint64. The return value follows the convention of // math/big. func (x Currency) Cmp64(y uint64) int { return x.Cmp(NewCurrency64(y)) } // Div returns a new Currency value c = x / y. func (x Currency) Div(y Currency) (c Currency) { c.i.Div(&x.i, &y.i) return } // Div64 returns a new Currency value c = x / y. func (x Currency) Div64(y uint64) (c Currency) { c.i.Div(&x.i, new(big.Int).SetUint64(y)) return } // Equals returns true if x and y have the same value. func (x Currency) Equals(y Currency) bool { return x.Cmp(y) == 0 } // Equals64 returns true if x and y have the same value. func (x Currency) Equals64(y uint64) bool { return x.Cmp64(y) == 0 } // Mul returns a new Currency value c = x * y. func (x Currency) Mul(y Currency) (c Currency) { c.i.Mul(&x.i, &y.i) return } // Mul64 returns a new Currency value c = x * y. func (x Currency) Mul64(y uint64) (c Currency) { c.i.Mul(&x.i, new(big.Int).SetUint64(y)) return } // COMPATv0.4.0 - until the first 10e3 blocks have been archived, MulFloat is // needed while verifying the first set of blocks. // // MulFloat returns a new Currency value y = c * x, where x is a float64. // Behavior is undefined when x is negative. func (x Currency) MulFloat(y float64) (c Currency) { if y < 0 { build.Critical(ErrNegativeCurrency) } else { cRat := new(big.Rat).Mul( new(big.Rat).SetInt(&x.i), new(big.Rat).SetFloat64(y), ) c.i.Div(cRat.Num(), cRat.Denom()) } return } // MulRat returns a new Currency value c = x * y, where y is a big.Rat. func (x Currency) MulRat(y *big.Rat) (c Currency) { if y.Sign() < 0 { build.Critical(ErrNegativeCurrency) } else { c.i.Mul(&x.i, y.Num()) c.i.Div(&c.i, y.Denom()) } return } // MulTax returns a new Currency value c = x * 0.039, where 0.039 is a big.Rat. func (x Currency) MulTax() (c Currency) { c.i.Mul(&x.i, big.NewInt(39)) c.i.Div(&c.i, big.NewInt(1000)) return c } // RoundDown returns the largest multiple of y <= x. func (x Currency) RoundDown(y Currency) (c Currency) { diff := new(big.Int).Mod(&x.i, &y.i) c.i.Sub(&x.i, diff) return } // IsZero returns true if the value is 0, false otherwise. func (c Currency) IsZero() bool { return c.i.Sign() <= 0 } // Sqrt returns a new Currency value y = sqrt(c). Result is rounded down to the // nearest integer. func (x Currency) Sqrt() (c Currency) { f, _ := new(big.Rat).SetInt(&x.i).Float64() sqrt := new(big.Rat).SetFloat64(math.Sqrt(f)) c.i.Div(sqrt.Num(), sqrt.Denom()) return } // Sub returns a new Currency value c = x - y. Behavior is undefined when // x < y. func (x Currency) Sub(y Currency) (c Currency) { if x.Cmp(y) < 0 { c = x build.Critical(ErrNegativeCurrency) } else { c.i.Sub(&x.i, &y.i) } return } // Uint64 converts a Currency to a uint64. An error is returned because this // function is sometimes called on values that can be determined by users - // rather than have all user-facing points do input checking, the input // checking should happen at the base type. This minimizes the chances of a // rogue user causing a build.Critical to be triggered. func (c Currency) Uint64() (u uint64, err error) { if c.Cmp(NewCurrency64(math.MaxUint64)) > 0 { return 0, ErrUint64Overflow } return c.Big().Uint64(), nil } // MarshalJSON implements the json.Marshaler interface. func (c Currency) MarshalJSON() ([]byte, error) { // Must enclosed the value in quotes; otherwise JS will convert it to a // double and lose precision. return []byte(`"` + c.String() + `"`), nil } // UnmarshalJSON implements the json.Unmarshaler interface. An error is // returned if a negative number is provided. func (c *Currency) UnmarshalJSON(b []byte) error { // UnmarshalJSON does not expect quotes b = bytes.Trim(b, `"`) err := c.i.UnmarshalJSON(b) if err != nil { return err } if c.i.Sign() < 0 { c.i = *big.NewInt(0) return ErrNegativeCurrency } return nil } // MarshalSia implements the encoding.SiaMarshaler interface. It writes the // byte-slice representation of the Currency's internal big.Int to w. Note // that as the bytes of the big.Int correspond to the absolute value of the // integer, there is no way to marshal a negative Currency. func (c Currency) MarshalSia(w io.Writer) error { return encoding.WritePrefix(w, c.i.Bytes()) } // UnmarshalSia implements the encoding.SiaUnmarshaler interface. func (c *Currency) UnmarshalSia(r io.Reader) error { b, err := encoding.ReadPrefix(r, 256) if err != nil { return err } c.i.SetBytes(b) return nil } // String implements the fmt.Stringer interface. func (c Currency) String() string { return c.i.String() } // Scan implements the fmt.Scanner interface, allowing Currency values to be // scanned from text. func (c *Currency) Scan(s fmt.ScanState, ch rune) error { err := c.i.Scan(s, ch) if err != nil { return err } if c.i.Sign() < 0 { return ErrNegativeCurrency } return nil } Sia-1.0.4/types/currency_test.go000066400000000000000000000274661302033070500166240ustar00rootroot00000000000000package types import ( "bytes" "fmt" "math" "math/big" "testing" "github.com/NebulousLabs/Sia/encoding" ) // TestNewCurrency initializes a standard new currency. func TestNewCurrency(t *testing.T) { b := big.NewInt(481) c := NewCurrency(b) if b.String() != c.String() { t.Error("NewCurrency does't seem to work properly") } } // TestCurrencyAdd probes the addition function of the currency type. func TestCurrencyAdd(t *testing.T) { c7 := NewCurrency64(7) c12 := NewCurrency64(12) c19 := NewCurrency64(19) if c7.Add(c12).Cmp(c19) != 0 { t.Error("Add doesn't seem to work right") } } // TestCurrencyToBig tests the Big method for the currency type func TestCurrencyToBig(t *testing.T) { c := NewCurrency64(125) cb := c.Big() b := big.NewInt(125) if b.Cmp(cb) != 0 { t.Error("currency to big has failed") } } // TestCurrencyCmp tests the Cmp method for the currency type func TestCurrencyCmp(t *testing.T) { tests := []struct { x, y Currency exp int }{ {NewCurrency64(0), NewCurrency64(0), 0}, {NewCurrency64(0), NewCurrency64(1), -1}, {NewCurrency64(1), NewCurrency64(0), 1}, {NewCurrency64(100), NewCurrency64(7), 1}, {NewCurrency64(777), NewCurrency(big.NewInt(777)), 0}, {NewCurrency(big.NewInt(7)), NewCurrency(big.NewInt(8)), -1}, } for _, test := range tests { if c := test.x.Cmp(test.y); c != test.exp { t.Errorf("expected %v.Cmp(%v) == %v, got %v", test.x, test.y, test.exp, c) } else if bc := test.x.Big().Cmp(test.y.Big()); c != bc { t.Errorf("Currency.Cmp (%v) does not match big.Int.Cmp (%v) for %v.Cmp(%v)", c, bc, test.x, test.y) } } } // TestCurrencyCmp64 tests the Cmp64 method for the currency type func TestCurrencyCmp64(t *testing.T) { tests := []struct { x Currency y uint64 exp int }{ {NewCurrency64(0), 0, 0}, {NewCurrency64(0), 1, -1}, {NewCurrency64(1), 0, 1}, {NewCurrency64(100), 7, 1}, {NewCurrency64(777), 777, 0}, {NewCurrency(big.NewInt(7)), 8, -1}, } for _, test := range tests { if c := test.x.Cmp64(test.y); c != test.exp { t.Errorf("expected %v.Cmp64(%v) == %v, got %v", test.x, test.y, test.exp, c) } else if bc := test.x.Big().Cmp(big.NewInt(int64(test.y))); c != bc { t.Errorf("Currency.Cmp64 (%v) does not match big.Int.Cmp (%v) for %v.Cmp64(%v)", c, bc, test.x, test.y) } } } // TestCurrencyDiv checks that the div function has been correctly implemented. func TestCurrencyDiv(t *testing.T) { c9 := NewCurrency64(9) c10 := NewCurrency64(10) c90 := NewCurrency64(90) c97 := NewCurrency64(97) c90D10 := c90.Div(c10) if c90D10.Cmp(c9) != 0 { t.Error("Dividing 90 by 10 should produce 9") } c97D10 := c97.Div(c10) if c97D10.Cmp(c9) != 0 { t.Error("Dividing 97 by 10 should produce 9") } } // TestCurrencyDiv64 checks that the Div64 function has been correctly implemented. func TestCurrencyDiv64(t *testing.T) { c9 := NewCurrency64(9) u10 := uint64(10) c90 := NewCurrency64(90) c97 := NewCurrency64(97) c90D10 := c90.Div64(u10) if c90D10.Cmp(c9) != 0 { t.Error("Dividing 90 by 10 should produce 9") } c97D10 := c97.Div64(u10) if c97D10.Cmp(c9) != 0 { t.Error("Dividing 97 by 10 should produce 9") } } // TestCurrencyEquals tests the Equals method for the currency type func TestCurrencyEquals(t *testing.T) { tests := []struct { x, y Currency exp bool }{ {NewCurrency64(0), NewCurrency64(0), true}, {NewCurrency64(0), NewCurrency64(1), false}, {NewCurrency64(1), NewCurrency64(0), false}, {NewCurrency64(100), NewCurrency64(7), false}, {NewCurrency64(777), NewCurrency(big.NewInt(777)), true}, {NewCurrency(big.NewInt(7)), NewCurrency(big.NewInt(8)), false}, } for _, test := range tests { if eq := test.x.Equals(test.y); eq != test.exp { t.Errorf("expected %v.Equals(%v) == %v, got %v", test.x, test.y, test.exp, eq) } else if bc := test.x.Big().Cmp(test.y.Big()); (bc == 0) != eq { t.Errorf("Currency.Equals (%v) does not match big.Int.Cmp (%v) for %v.Equals(%v)", eq, bc, test.x, test.y) } } } // TestCurrencyEquals64 tests the Equals64 method for the currency type func TestCurrencyEquals64(t *testing.T) { tests := []struct { x Currency y uint64 exp bool }{ {NewCurrency64(0), 0, true}, {NewCurrency64(0), 1, false}, {NewCurrency64(1), 0, false}, {NewCurrency64(100), 7, false}, {NewCurrency64(777), 777, true}, {NewCurrency(big.NewInt(7)), 8, false}, } for _, test := range tests { if eq := test.x.Equals64(test.y); eq != test.exp { t.Errorf("expected %v.Equals64(%v) == %v, got %v", test.x, test.y, test.exp, eq) } else if bc := test.x.Big().Cmp(big.NewInt(int64(test.y))); (bc == 0) != eq { t.Errorf("Currency.Equals64 (%v) does not match big.Int.Cmp (%v) for %v.Equals64(%v)", eq, bc, test.x, test.y) } } } // TestCurrencyMul probes the Mul function of the currency type. func TestCurrencyMul(t *testing.T) { c5 := NewCurrency64(5) c6 := NewCurrency64(6) c30 := NewCurrency64(30) if c5.Mul(c6).Cmp(c30) != 0 { t.Error("Multiplying 5 by 6 should equal 30") } } // TestCurrencyMul64 probes the Mul64 function of the currency type. func TestCurrencyMul64(t *testing.T) { c5 := NewCurrency64(5) u6 := uint64(6) c30 := NewCurrency64(30) if c5.Mul64(u6).Cmp(c30) != 0 { t.Error("Multiplying 5 by 6 should equal 30") } } // TestCurrencyMulRat probes the MulRat function of the currency type. func TestCurrencyMulRat(t *testing.T) { c5 := NewCurrency64(5) c7 := NewCurrency64(7) c10 := NewCurrency64(10) if c5.MulRat(big.NewRat(2, 1)).Cmp(c10) != 0 { t.Error("Multiplying 5 by 2 should return 10") } if c5.MulRat(big.NewRat(3, 2)).Cmp(c7) != 0 { t.Error("Multiplying 5 by 1.5 should return 7") } } // TestCurrencyRoundDown probes the RoundDown function of the currency type. func TestCurrencyRoundDown(t *testing.T) { // 10,000 is chosen because that's how many siafunds there usually are. c40000 := NewCurrency64(40000) c45000 := NewCurrency64(45000) if c45000.RoundDown(NewCurrency64(10000)).Cmp(c40000) != 0 { t.Error("rounding down 45000 to the nearest 10000 didn't work") } } // TestCurrencyIsZero probes the IsZero function of the currency type. func TestCurrencyIsZero(t *testing.T) { c0 := NewCurrency64(0) c1 := NewCurrency64(1) if !c0.IsZero() { t.Error("IsZero returns wrong value for 0") } if c1.IsZero() { t.Error("IsZero returns wrong value for 1") } } // TestCurrencySqrt probes the Sqrt function of the currency type. func TestCurrencySqrt(t *testing.T) { c8 := NewCurrency64(8) c64 := NewCurrency64(64) c80 := NewCurrency64(80) sqrt64 := c64.Sqrt() sqrt80 := c80.Sqrt() if c8.Cmp(sqrt64) != 0 { t.Error("square root of 64 should be 8") } if c8.Cmp(sqrt80) != 0 { t.Error("square root of 80 should be 8") } } // TestCurrencySub probes the Sub function of the currency type. func TestCurrencySub(t *testing.T) { c3 := NewCurrency64(3) c13 := NewCurrency64(13) c16 := NewCurrency64(16) if c16.Sub(c3).Cmp(c13) != 0 { t.Error("16 minus 3 should equal 13") } } // TestCurrencyMarshalJSON probes the MarshalJSON and UnmarshalJSON functions // of the currency type. func TestCurrencyMarshalJSON(t *testing.T) { b30 := big.NewInt(30) c30 := NewCurrency64(30) bMar30, err := b30.MarshalJSON() if err != nil { t.Fatal(err) } cMar30, err := c30.MarshalJSON() if err != nil { t.Fatal(err) } if !bytes.Equal(bMar30, bytes.Trim(cMar30, `"`)) { t.Error("Currency does not match the marshalling of its math/big equivalent") } var cUmar30 Currency err = cUmar30.UnmarshalJSON(cMar30) if err != nil { t.Fatal(err) } if c30.Cmp(cUmar30) != 0 { t.Error("Incorrect unmarshalling of currency type.") } cMar30[0] = 0 err = cUmar30.UnmarshalJSON(cMar30) if err == nil { t.Error("JSON decoded nonsense input") } } // TestCurrencyMarshalSia probes the MarshalSia and UnmarshalSia functions of // the currency type. func TestCurrencyMarshalSia(t *testing.T) { c := NewCurrency64(1656) buf := new(bytes.Buffer) err := c.MarshalSia(buf) if err != nil { t.Fatal(err) } var cUmar Currency cUmar.UnmarshalSia(buf) if c.Cmp(cUmar) != 0 { t.Error("marshal and unmarshal mismatch for currency type") } } // TestCurrencyString probes the String function of the currency type. func TestCurrencyString(t *testing.T) { b := big.NewInt(7135) c := NewCurrency64(7135) if b.String() != c.String() { t.Error("string function not behaving as expected") } } // TestCurrencyScan probes the Scan function of the currency type. func TestCurrencyScan(t *testing.T) { var c0 Currency c1 := NewCurrency64(81293) _, err := fmt.Sscan("81293", &c0) if err != nil { t.Fatal(err) } if c0.Cmp(c1) != 0 { t.Error("scanned number does not equal expected value") } _, err = fmt.Sscan("z", &c0) if err == nil { t.Fatal("scan is accepting garbage input") } } // TestCurrencyEncoding checks that a currency can encode and decode without // error. func TestCurrencyEncoding(t *testing.T) { c := NewCurrency64(351) cMar := encoding.Marshal(c) var cUmar Currency err := encoding.Unmarshal(cMar, &cUmar) if err != nil { t.Error("Error unmarshalling a currency:", err) } if cUmar.Cmp(c) != 0 { t.Error("Marshalling and Unmarshalling a currency did not work correctly") } } // TestNegativeCurrencyMulRat checks that negative numbers are rejected when // calling MulRat on the currency type. func TestNegativeCurrencyMulRat(t *testing.T) { // In debug mode, attempting to get a negative currency results in a panic. defer func() { r := recover() if r == nil { t.Error("no panic occurred when trying to create a negative currency") } }() c := NewCurrency64(12) _ = c.MulRat(big.NewRat(-1, 1)) } // TestNegativeCurrencySub checks that negative numbers are prevented when // using subtraction on the currency type. func TestNegativeCurrencySub(t *testing.T) { // In debug mode, attempting to get a negative currency results in a panic. defer func() { r := recover() if r == nil { t.Error("no panic occurred when trying to create a negative currency") } }() c1 := NewCurrency64(1) c2 := NewCurrency64(2) _ = c1.Sub(c2) } // TestNegativeCurrencyUnmarshalJSON tries to unmarshal a negative number from // JSON. func TestNegativeCurrencyUnmarshalJSON(t *testing.T) { // Marshal a 2 digit number. c := NewCurrency64(35) cMar, err := c.MarshalJSON() if err != nil { t.Fatal(err) } // Change the first digit to a negative character. cMar[0] = 45 // Try unmarshalling the negative currency. var cNeg Currency err = cNeg.UnmarshalJSON(cMar) if err != ErrNegativeCurrency { t.Error("expecting ErrNegativeCurrency:", err) } if cNeg.i.Sign() < 0 { t.Error("negative currency returned") } } // TestNegativeCurrencyScan tries to scan in a negative number and checks for // an error. func TestNegativeCurrencyScan(t *testing.T) { var c Currency _, err := fmt.Sscan("-23", &c) if err != ErrNegativeCurrency { t.Error("expecting ErrNegativeCurrency:", err) } } // TestNegativeCurrencies tries an array of ways to produce a negative currency. func TestNegativeNewCurrency(t *testing.T) { // In debug mode, attempting to get a negative currency results in a panic. defer func() { r := recover() if r == nil { t.Error("no panic occurred when trying to create a negative currency") } }() // Try to create a new currency from a negative number. negBig := big.NewInt(-1) _ = NewCurrency(negBig) } // TestCurrencyUint64 tests that a currency is correctly converted to a uint64. func TestCurrencyUint64(t *testing.T) { // Try a set of valid values. values := []uint64{0, 1, 2, 3, 4, 25e3, math.MaxUint64 - 1e6, math.MaxUint64} for _, value := range values { c := NewCurrency64(value) result, err := c.Uint64() if err != nil { t.Error(err) } if value != result { t.Error("uint64 conversion failed") } } // Try an overflow. c := NewCurrency64(math.MaxUint64) c = c.Mul(NewCurrency64(2)) result, err := c.Uint64() if err != ErrUint64Overflow { t.Error(err) } if result != 0 { t.Error("result is not being zeroed in the event of an error") } } Sia-1.0.4/types/filecontracts.go000066400000000000000000000130211302033070500165510ustar00rootroot00000000000000package types // filecontracts.go contains the basic structs and helper functions for file // contracts. import ( "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" ) var ( ProofValid ProofStatus = true ProofMissed ProofStatus = false ) type ( // A FileContract is a public record of a storage agreement between a "host" // and a "renter." It mandates that a host must submit a storage proof to the // network, proving that they still possess the file they have agreed to // store. // // The party must submit the storage proof in a block that is between // 'WindowStart' and 'WindowEnd'. Upon submitting the proof, the outputs // for 'ValidProofOutputs' are created. If the party does not submit a // storage proof by 'WindowEnd', then the outputs for 'MissedProofOutputs' // are created instead. The sum of 'MissedProofOutputs' must equal // 'Payout', and the sum of 'ValidProofOutputs' must equal 'Payout' plus // the siafund fee. This fee is sent to the siafund pool, which is a set // of siacoins only spendable by siafund owners. // // Under normal circumstances, the payout will be funded by both the host and // the renter, which gives the host incentive not to lose the file. The // 'ValidProofUnlockHash' will typically be spendable by host, and the // 'MissedProofUnlockHash' will either by spendable by the renter or by // nobody (the ZeroUnlockHash). // // A contract can be terminated early by submitting a FileContractTermination // whose UnlockConditions hash to 'TerminationHash'. FileContract struct { FileSize uint64 `json:"filesize"` FileMerkleRoot crypto.Hash `json:"filemerkleroot"` WindowStart BlockHeight `json:"windowstart"` WindowEnd BlockHeight `json:"windowend"` Payout Currency `json:"payout"` ValidProofOutputs []SiacoinOutput `json:"validproofoutputs"` MissedProofOutputs []SiacoinOutput `json:"missedproofoutputs"` UnlockHash UnlockHash `json:"unlockhash"` RevisionNumber uint64 `json:"revisionnumber"` } // A FileContractRevision revises an existing file contract. The ParentID // points to the file contract that is being revised. The UnlockConditions // are the conditions under which the revision is valid, and must match the // UnlockHash of the parent file contract. The Payout of the file contract // cannot be changed, but all other fields are allowed to be changed. The // sum of the outputs must match the original payout (taking into account // the fee for the proof payouts.) A revision number is included. When // getting accepted, the revision number of the revision must be higher // than any previously seen revision number for that file contract. // // FileContractRevisions enable trust-free modifications to existing file // contracts. FileContractRevision struct { ParentID FileContractID `json:"parentid"` UnlockConditions UnlockConditions `json:"unlockconditions"` NewRevisionNumber uint64 `json:"newrevisionnumber"` NewFileSize uint64 `json:"newfilesize"` NewFileMerkleRoot crypto.Hash `json:"newfilemerkleroot"` NewWindowStart BlockHeight `json:"newwindowstart"` NewWindowEnd BlockHeight `json:"newwindowend"` NewValidProofOutputs []SiacoinOutput `json:"newvalidproofoutputs"` NewMissedProofOutputs []SiacoinOutput `json:"newmissedproofoutputs"` NewUnlockHash UnlockHash `json:"newunlockhash"` } // A StorageProof fulfills a FileContract. The proof contains a specific // segment of the file, along with a set of hashes from the file's Merkle // tree. In combination, these can be used to prove that the segment came // from the file. To prevent abuse, the segment must be chosen randomly, so // the ID of block 'WindowStart' - 1 is used as a seed value; see // StorageProofSegment for the exact implementation. // // A transaction with a StorageProof cannot have any SiacoinOutputs, // SiafundOutputs, or FileContracts. This is because a mundane reorg can // invalidate the proof, and with it the rest of the transaction. StorageProof struct { ParentID FileContractID `json:"parentid"` Segment [crypto.SegmentSize]byte `json:"segment"` HashSet []crypto.Hash `json:"hashset"` } ProofStatus bool ) // StorageProofOutputID returns the ID of an output created by a file // contract, given the status of the storage proof. The ID is calculating by // hashing the concatenation of the StorageProofOutput Specifier, the ID of // the file contract that the proof is for, a boolean indicating whether the // proof was valid (true) or missed (false), and the index of the output // within the file contract. func (fcid FileContractID) StorageProofOutputID(proofStatus ProofStatus, i uint64) SiacoinOutputID { return SiacoinOutputID(crypto.HashAll( SpecifierStorageProofOutput, fcid, proofStatus, i, )) } // PostTax returns the amount of currency remaining in a file contract payout // after tax. func PostTax(height BlockHeight, payout Currency) Currency { return payout.Sub(Tax(height, payout)) } // Tax returns the amount of Currency that will be taxed from fc. func Tax(height BlockHeight, payout Currency) Currency { // COMPATv0.4.0 - until the first 20,000 blocks have been archived, they // will need to be handled in a special way. if (height < 21e3 && build.Release == "standard") || (height < 10 && build.Release == "testing") { return payout.MulFloat(0.039).RoundDown(SiafundCount) } return payout.MulTax().RoundDown(SiafundCount) } Sia-1.0.4/types/filecontracts_test.go000066400000000000000000000017631302033070500176220ustar00rootroot00000000000000package types import ( "testing" ) // TestFileContractTax probes the Tax function. func TestTax(t *testing.T) { // Test explicit values for post-hardfork tax values. if Tax(1e9, NewCurrency64(125e9)).Cmp(NewCurrency64(4875e6)) != 0 { t.Error("tax is being calculated incorrectly") } if PostTax(1e9, NewCurrency64(125e9)).Cmp(NewCurrency64(120125e6)) != 0 { t.Error("tax is being calculated incorrectly") } // Test equivalency for a series of values. if testing.Short() { t.SkipNow() } // COMPATv0.4.0 - check at height 0. for i := uint64(0); i < 10e3; i++ { val := NewCurrency64((1e3 * i) + i) tax := Tax(0, val) postTax := PostTax(0, val) if val.Cmp(tax.Add(postTax)) != 0 { t.Error("tax calculation inconsistent for", i) } } // Check at height 1e9 for i := uint64(0); i < 10e3; i++ { val := NewCurrency64((1e3 * i) + i) tax := Tax(1e9, val) postTax := PostTax(1e9, val) if val.Cmp(tax.Add(postTax)) != 0 { t.Error("tax calculation inconsistent for", i) } } } Sia-1.0.4/types/siag0of1of1.siakey000066400000000000000000000002571302033070500166170ustar00rootroot00000000000000siag1.0hMw(T-' ΥfYD62>(5!("ą1,ލJ ed25519 2>(5!("ą1,ލJ Sia-1.0.4/types/siag0of2of3.siakey000066400000000000000000000004371302033070500166220ustar00rootroot00000000000000siag1.0آqiq#wuTgOەv;bq~<;M(pι0ed25519 v;bq~<;M(pι0ed25519 ij})\jo;4Q^a*ed25519 ^3*>Qr3l|ǩ*% HvSia-1.0.4/types/siag1of2of3.siakey000066400000000000000000000004371302033070500166230ustar00rootroot00000000000000siag1.0_^bjBAgU“,8oij})\jo;4Q^a*ed25519 v;bq~<;M(pι0ed25519 ij})\jo;4Q^a*ed25519 ^3*>Qr3l|ǩ*% HvSia-1.0.4/types/siag2of2of3.siakey000066400000000000000000000004371302033070500166240ustar00rootroot00000000000000siag1.0~BV^UW2wVCb:WZ-'F ^3*>Qr3l|ǩ*% Hved25519 v;bq~<;M(pι0ed25519 ij})\jo;4Q^a*ed25519 ^3*>Qr3l|ǩ*% HvSia-1.0.4/types/signatures.go000066400000000000000000000350761302033070500161130ustar00rootroot00000000000000package types // signatures.go contains all of the types and functions related to creating // and verifying transaction signatures. There are a lot of rules surrounding // the correct use of signatures. Signatures can cover part or all of a // transaction, can be multiple different algorithms, and must satify a field // called 'UnlockConditions'. import ( "errors" "fmt" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" ) var ( // These Specifiers enumerate the types of signatures that are recognized // by this implementation. If a signature's type is unrecognized, the // signature is treated as valid. Signatures using the special "entropy" // type are always treated as invalid; see Consensus.md for more details. SignatureEntropy = Specifier{'e', 'n', 't', 'r', 'o', 'p', 'y'} SignatureEd25519 = Specifier{'e', 'd', '2', '5', '5', '1', '9'} ErrEntropyKey = errors.New("transaction tries to sign an entproy public key") ErrFrivilousSignature = errors.New("transaction contains a frivilous siganture") ErrInvalidPubKeyIndex = errors.New("transaction contains a signature that points to a nonexistent public key") ErrInvalidUnlockHashChecksum = errors.New("provided unlock hash has an invalid checksum") ErrMissingSignatures = errors.New("transaction has inputs with missing signatures") ErrPrematureSignature = errors.New("timelock on signature has not expired") ErrPublicKeyOveruse = errors.New("public key was used multiple times while signing transaction") ErrSortedUniqueViolation = errors.New("sorted unique violation") ErrUnlockHashWrongLen = errors.New("marshalled unlock hash is the wrong length") ErrWholeTransactionViolation = errors.New("covered fields violation") // FullCoveredFields is a covered fileds object where the // 'WholeTransaction' field has been set to true. The primary purpose of // this variable is syntactic sugar. FullCoveredFields = CoveredFields{WholeTransaction: true} ) type ( // CoveredFields indicates which fields in a transaction have been covered by // the signature. (Note that the signature does not sign the fields // themselves, but rather their combined hash; see SigHash.) Each slice // corresponds to a slice in the Transaction type, indicating which indices of // the slice have been signed. The indices must be valid, i.e. within the // bounds of the slice. In addition, they must be sorted and unique. // // As a convenience, a signature of the entire transaction can be indicated by // the 'WholeTransaction' field. If 'WholeTransaction' == true, all other // fields must be empty (except for the Signatures field, since a signature // cannot sign itself). CoveredFields struct { WholeTransaction bool `json:"wholetransaction"` SiacoinInputs []uint64 `json:"siacoininputs"` SiacoinOutputs []uint64 `json:"siacoinoutputs"` FileContracts []uint64 `json:"filecontracts"` FileContractRevisions []uint64 `json:"filecontractrevisions"` StorageProofs []uint64 `json:"storageproofs"` SiafundInputs []uint64 `json:"siafundinputs"` SiafundOutputs []uint64 `json:"siafundoutputs"` MinerFees []uint64 `json:"minerfees"` ArbitraryData []uint64 `json:"arbitrarydata"` TransactionSignatures []uint64 `json:"transactionsignatures"` } // A SiaPublicKey is a public key prefixed by a Specifier. The Specifier // indicates the algorithm used for signing and verification. Unrecognized // algorithms will always verify, which allows new algorithms to be added to // the protocol via a soft-fork. SiaPublicKey struct { Algorithm Specifier `json:"algorithm"` Key []byte `json:"key"` } // A TransactionSignature is a signature that is included in the transaction. // The signature should correspond to a public key in one of the // UnlockConditions of the transaction. This key is specified first by // 'ParentID', which specifies the UnlockConditions, and then // 'PublicKeyIndex', which indicates the key in the UnlockConditions. There // are three types that use UnlockConditions: SiacoinInputs, SiafundInputs, // and FileContractTerminations. Each of these types also references a // ParentID, and this is the hash that 'ParentID' must match. The 'Timelock' // prevents the signature from being used until a certain height. // 'CoveredFields' indicates which parts of the transaction are being signed; // see CoveredFields. TransactionSignature struct { ParentID crypto.Hash `json:"parentid"` PublicKeyIndex uint64 `json:"publickeyindex"` Timelock BlockHeight `json:"timelock"` CoveredFields CoveredFields `json:"coveredfields"` Signature []byte `json:"signature"` } // UnlockConditions are a set of conditions which must be met to execute // certain actions, such as spending a SiacoinOutput or terminating a // FileContract. // // The simplest requirement is that the block containing the UnlockConditions // must have a height >= 'Timelock'. // // 'PublicKeys' specifies the set of keys that can be used to satisfy the // UnlockConditions; of these, at least 'SignaturesRequired' unique keys must sign // the transaction. The keys that do not need to use the same cryptographic // algorithm. // // If 'SignaturesRequired' == 0, the UnlockConditions are effectively "anyone can // unlock." If 'SignaturesRequired' > len('PublicKeys'), then the UnlockConditions // cannot be fulfilled under any circumstances. UnlockConditions struct { Timelock BlockHeight `json:"timelock"` PublicKeys []SiaPublicKey `json:"publickeys"` SignaturesRequired uint64 `json:"signaturesrequired"` } // Each input has a list of public keys and a required number of signatures. // inputSignatures keeps track of which public keys have been used and how many // more signatures are needed. inputSignatures struct { remainingSignatures uint64 possibleKeys []SiaPublicKey usedKeys map[uint64]struct{} index int } ) // UnlockHash calculates the root hash of a Merkle tree of the // UnlockConditions object. The leaves of this tree are formed by taking the // hash of the timelock, the hash of the public keys (one leaf each), and the // hash of the number of signatures. The keys are put in the middle because // Timelock and SignaturesRequired are both low entropy fields; they can be // protected by having random public keys next to them. func (uc UnlockConditions) UnlockHash() UnlockHash { tree := crypto.NewTree() tree.PushObject(uc.Timelock) for i := range uc.PublicKeys { tree.PushObject(uc.PublicKeys[i]) } tree.PushObject(uc.SignaturesRequired) return UnlockHash(tree.Root()) } // SigHash returns the hash of the fields in a transaction covered by a given // signature. See CoveredFields for more details. func (t Transaction) SigHash(i int) crypto.Hash { cf := t.TransactionSignatures[i].CoveredFields var signedData []byte if cf.WholeTransaction { signedData = encoding.MarshalAll( t.SiacoinInputs, t.SiacoinOutputs, t.FileContracts, t.FileContractRevisions, t.StorageProofs, t.SiafundInputs, t.SiafundOutputs, t.MinerFees, t.ArbitraryData, t.TransactionSignatures[i].ParentID, t.TransactionSignatures[i].PublicKeyIndex, t.TransactionSignatures[i].Timelock, ) } else { for _, input := range cf.SiacoinInputs { signedData = append(signedData, encoding.Marshal(t.SiacoinInputs[input])...) } for _, output := range cf.SiacoinOutputs { signedData = append(signedData, encoding.Marshal(t.SiacoinOutputs[output])...) } for _, contract := range cf.FileContracts { signedData = append(signedData, encoding.Marshal(t.FileContracts[contract])...) } for _, revision := range cf.FileContractRevisions { signedData = append(signedData, encoding.Marshal(t.FileContractRevisions[revision])...) } for _, storageProof := range cf.StorageProofs { signedData = append(signedData, encoding.Marshal(t.StorageProofs[storageProof])...) } for _, siafundInput := range cf.SiafundInputs { signedData = append(signedData, encoding.Marshal(t.SiafundInputs[siafundInput])...) } for _, siafundOutput := range cf.SiafundOutputs { signedData = append(signedData, encoding.Marshal(t.SiafundOutputs[siafundOutput])...) } for _, minerFee := range cf.MinerFees { signedData = append(signedData, encoding.Marshal(t.MinerFees[minerFee])...) } for _, arbData := range cf.ArbitraryData { signedData = append(signedData, encoding.Marshal(t.ArbitraryData[arbData])...) } } for _, sig := range cf.TransactionSignatures { signedData = append(signedData, encoding.Marshal(t.TransactionSignatures[sig])...) } return crypto.HashBytes(signedData) } // sortedUnique checks that 'elems' is sorted, contains no repeats, and that no // element is larger than or equal to 'max'. func sortedUnique(elems []uint64, max int) bool { if len(elems) == 0 { return true } biggest := elems[0] for _, elem := range elems[1:] { if elem <= biggest { return false } biggest = elem } if biggest >= uint64(max) { return false } return true } // validCoveredFields makes sure that all covered fields objects in the // signatures follow the rules. This means that if 'WholeTransaction' is set to // true, all fields except for 'Signatures' must be empty. All fields must be // sorted numerically, and there can be no repeats. func (t Transaction) validCoveredFields() error { for _, sig := range t.TransactionSignatures { // convenience variables cf := sig.CoveredFields fieldMaxs := []struct { field []uint64 max int }{ {cf.SiacoinInputs, len(t.SiacoinInputs)}, {cf.SiacoinOutputs, len(t.SiacoinOutputs)}, {cf.FileContracts, len(t.FileContracts)}, {cf.FileContractRevisions, len(t.FileContractRevisions)}, {cf.StorageProofs, len(t.StorageProofs)}, {cf.SiafundInputs, len(t.SiafundInputs)}, {cf.SiafundOutputs, len(t.SiafundOutputs)}, {cf.MinerFees, len(t.MinerFees)}, {cf.ArbitraryData, len(t.ArbitraryData)}, {cf.TransactionSignatures, len(t.TransactionSignatures)}, } // Check that all fields are empty if 'WholeTransaction' is set, except // for the Signatures field which isn't affected. if cf.WholeTransaction { // 'WholeTransaction' does not check signatures. for _, fieldMax := range fieldMaxs[:len(fieldMaxs)-1] { if len(fieldMax.field) != 0 { return ErrWholeTransactionViolation } } } // Check that all fields are sorted, and without repeat values, and // that all elements point to objects that exists within the // transaction. If there are repeats, it means a transaction is trying // to sign the same object twice. This is unncecessary, and opens up a // DoS vector where the transaction asks the verifier to verify many GB // of data. for _, fieldMax := range fieldMaxs { if !sortedUnique(fieldMax.field, fieldMax.max) { return ErrSortedUniqueViolation } } } return nil } // validSignatures checks the validaty of all signatures in a transaction. func (t *Transaction) validSignatures(currentHeight BlockHeight) error { // Check that all covered fields objects follow the rules. err := t.validCoveredFields() if err != nil { return err } // Create the inputSignatures object for each input. sigMap := make(map[crypto.Hash]*inputSignatures) for i, input := range t.SiacoinInputs { id := crypto.Hash(input.ParentID) _, exists := sigMap[id] if exists { return ErrDoubleSpend } sigMap[id] = &inputSignatures{ remainingSignatures: input.UnlockConditions.SignaturesRequired, possibleKeys: input.UnlockConditions.PublicKeys, usedKeys: make(map[uint64]struct{}), index: i, } } for i, revision := range t.FileContractRevisions { id := crypto.Hash(revision.ParentID) _, exists := sigMap[id] if exists { return ErrDoubleSpend } sigMap[id] = &inputSignatures{ remainingSignatures: revision.UnlockConditions.SignaturesRequired, possibleKeys: revision.UnlockConditions.PublicKeys, usedKeys: make(map[uint64]struct{}), index: i, } } for i, input := range t.SiafundInputs { id := crypto.Hash(input.ParentID) _, exists := sigMap[id] if exists { return ErrDoubleSpend } sigMap[id] = &inputSignatures{ remainingSignatures: input.UnlockConditions.SignaturesRequired, possibleKeys: input.UnlockConditions.PublicKeys, usedKeys: make(map[uint64]struct{}), index: i, } } // Check all of the signatures for validity. for i, sig := range t.TransactionSignatures { // Check that sig corresponds to an entry in sigMap. inSig, exists := sigMap[crypto.Hash(sig.ParentID)] if !exists || inSig.remainingSignatures == 0 { return ErrFrivilousSignature } // Check that sig's key hasn't already been used. _, exists = inSig.usedKeys[sig.PublicKeyIndex] if exists { return ErrPublicKeyOveruse } // Check that the public key index refers to an existing public key. if sig.PublicKeyIndex >= uint64(len(inSig.possibleKeys)) { return ErrInvalidPubKeyIndex } // Check that the timelock has expired. if sig.Timelock > currentHeight { return ErrPrematureSignature } // Check that the signature verifies. Multiple signature schemes are // supported. publicKey := inSig.possibleKeys[sig.PublicKeyIndex] switch publicKey.Algorithm { case SignatureEntropy: // Entropy cannot ever be used to sign a transaction. return ErrEntropyKey case SignatureEd25519: // Decode the public key and signature. var edPK crypto.PublicKey err := encoding.Unmarshal([]byte(publicKey.Key), &edPK) if err != nil { return err } var edSig [crypto.SignatureSize]byte err = encoding.Unmarshal([]byte(sig.Signature), &edSig) if err != nil { return err } cryptoSig := crypto.Signature(edSig) sigHash := t.SigHash(i) err = crypto.VerifyHash(sigHash, edPK, cryptoSig) if err != nil { return err } default: // If the identifier is not recognized, assume that the signature // is valid. This allows more signature types to be added via soft // forking. } inSig.usedKeys[sig.PublicKeyIndex] = struct{}{} inSig.remainingSignatures-- } // Check that all inputs have been sufficiently signed. for _, reqSigs := range sigMap { if reqSigs.remainingSignatures != 0 { return ErrMissingSignatures } } return nil } // String defines how to print a SiaPublicKey - hex is used to keep things // compact during logging. The key type prefix and lack of a checksum help to // separate it from a sia address. func (spk *SiaPublicKey) String() string { return spk.Algorithm.String() + ":" + fmt.Sprintf("%x", spk.Key) } Sia-1.0.4/types/signatures_test.go000066400000000000000000000255711302033070500171510ustar00rootroot00000000000000package types import ( "testing" "github.com/NebulousLabs/Sia/crypto" ) // TestUnlockHash runs the UnlockHash code. func TestUnlockHash(t *testing.T) { uc := UnlockConditions{ Timelock: 1, PublicKeys: []SiaPublicKey{ { Algorithm: SignatureEntropy, Key: []byte{'f', 'a', 'k', 'e'}, }, }, SignaturesRequired: 3, } _ = uc.UnlockHash() } // TestSigHash runs the SigHash function of the transaction type. func TestSigHash(t *testing.T) { txn := Transaction{ SiacoinInputs: []SiacoinInput{{}}, SiacoinOutputs: []SiacoinOutput{{}}, FileContracts: []FileContract{{}}, FileContractRevisions: []FileContractRevision{{}}, StorageProofs: []StorageProof{{}}, SiafundInputs: []SiafundInput{{}}, SiafundOutputs: []SiafundOutput{{}}, MinerFees: []Currency{{}}, ArbitraryData: [][]byte{{'o'}, {'t'}}, TransactionSignatures: []TransactionSignature{ { CoveredFields: CoveredFields{ WholeTransaction: true, }, }, { CoveredFields: CoveredFields{ SiacoinInputs: []uint64{0}, SiacoinOutputs: []uint64{0}, FileContracts: []uint64{0}, FileContractRevisions: []uint64{0}, StorageProofs: []uint64{0}, SiafundInputs: []uint64{0}, SiafundOutputs: []uint64{0}, MinerFees: []uint64{0}, ArbitraryData: []uint64{0}, TransactionSignatures: []uint64{0}, }, }, }, } _ = txn.SigHash(0) _ = txn.SigHash(1) } // TestSortedUnique probes the sortedUnique function. func TestSortedUnique(t *testing.T) { su := []uint64{3, 5, 6, 8, 12} if !sortedUnique(su, 13) { t.Error("sortedUnique rejected a valid array") } if sortedUnique(su, 12) { t.Error("sortedUnique accepted an invalid max") } if sortedUnique(su, 11) { t.Error("sortedUnique accepted an invalid max") } unsorted := []uint64{3, 5, 3} if sortedUnique(unsorted, 6) { t.Error("sortedUnique accepted an unsorted array") } repeats := []uint64{2, 4, 4, 7} if sortedUnique(repeats, 8) { t.Error("sortedUnique accepted an array with repeats") } bothFlaws := []uint64{2, 3, 4, 5, 6, 6, 4} if sortedUnique(bothFlaws, 7) { t.Error("Sorted unique accetped array with multiple flaws") } } // TestTransactionValidCoveredFields probes the validCoveredFields menthod of // the transaction type. func TestTransactionValidCoveredFields(t *testing.T) { if testing.Short() { t.SkipNow() } // Create a transaction with all fields filled in minimally. The first // check has a legal CoveredFields object with 'WholeTransaction' set. txn := Transaction{ SiacoinInputs: []SiacoinInput{{}}, SiacoinOutputs: []SiacoinOutput{{}}, FileContracts: []FileContract{{}}, FileContractRevisions: []FileContractRevision{{}}, StorageProofs: []StorageProof{{}}, SiafundInputs: []SiafundInput{{}}, SiafundOutputs: []SiafundOutput{{}}, MinerFees: []Currency{{}}, ArbitraryData: [][]byte{{'o'}, {'t'}}, TransactionSignatures: []TransactionSignature{ { CoveredFields: CoveredFields{ WholeTransaction: true, }, }, }, } err := txn.validCoveredFields() if err != nil { t.Error(err) } // Second check has CoveredFields object where 'WholeTransaction' is not // set. txn.TransactionSignatures = append(txn.TransactionSignatures, TransactionSignature{ CoveredFields: CoveredFields{ SiacoinOutputs: []uint64{0}, MinerFees: []uint64{0}, ArbitraryData: []uint64{0}, FileContractRevisions: []uint64{0}, }, }) err = txn.validCoveredFields() if err != nil { t.Error(err) } // Add signature coverage to the first signature. This should not violate // any rules. txn.TransactionSignatures[0].CoveredFields.TransactionSignatures = []uint64{1} err = txn.validCoveredFields() if err != nil { t.Error(err) } // Add siacoin output coverage to the first signature. This should violate // rules, as the fields are not allowed to be set when 'WholeTransaction' // is set. txn.TransactionSignatures[0].CoveredFields.SiacoinOutputs = []uint64{0} err = txn.validCoveredFields() if err != ErrWholeTransactionViolation { t.Error("Expecting ErrWholeTransactionViolation, got", err) } // Create a SortedUnique violation instead of a WholeTransactionViolation. txn.TransactionSignatures[0].CoveredFields.SiacoinOutputs = nil txn.TransactionSignatures[0].CoveredFields.TransactionSignatures = []uint64{1, 2} err = txn.validCoveredFields() if err != ErrSortedUniqueViolation { t.Error("Expecting ErrSortedUniqueViolation, got", err) } } // TestTransactionValidSignatures probes the validSignatures method of the // Transaction type. func TestTransactionValidSignatures(t *testing.T) { // Create keys for use in signing and verifying. sk, pk, err := crypto.GenerateKeyPair() if err != nil { t.Fatal(err) } // Create UnlockConditions with 3 keys, 2 of which are required. The first // possible key is a standard signature. The second key is an unknown // signature type, which should always be accepted. The final type is an // entropy type, which should never be accepted. uc := UnlockConditions{ PublicKeys: []SiaPublicKey{ {Algorithm: SignatureEd25519, Key: pk[:]}, {}, {Algorithm: SignatureEntropy}, }, SignaturesRequired: 2, } // Create a transaction with each type of unlock condition. txn := Transaction{ SiacoinInputs: []SiacoinInput{ {UnlockConditions: uc}, }, FileContractRevisions: []FileContractRevision{ {UnlockConditions: uc}, }, SiafundInputs: []SiafundInput{ {UnlockConditions: uc}, }, } txn.FileContractRevisions[0].ParentID[0] = 1 // can't overlap with other objects txn.SiafundInputs[0].ParentID[0] = 2 // can't overlap with other objects // Create the signatures that spend the output. txn.TransactionSignatures = []TransactionSignature{ // First signatures use cryptography. { Timelock: 5, CoveredFields: CoveredFields{WholeTransaction: true}, }, { CoveredFields: CoveredFields{WholeTransaction: true}, }, { CoveredFields: CoveredFields{WholeTransaction: true}, }, // The second signatures should always work for being unrecognized // types. {PublicKeyIndex: 1}, {PublicKeyIndex: 1}, {PublicKeyIndex: 1}, } txn.TransactionSignatures[1].ParentID[0] = 1 txn.TransactionSignatures[2].ParentID[0] = 2 txn.TransactionSignatures[4].ParentID[0] = 1 txn.TransactionSignatures[5].ParentID[0] = 2 sigHash0 := txn.SigHash(0) sigHash1 := txn.SigHash(1) sigHash2 := txn.SigHash(2) sig0, err := crypto.SignHash(sigHash0, sk) if err != nil { t.Fatal(err) } sig1, err := crypto.SignHash(sigHash1, sk) if err != nil { t.Fatal(err) } sig2, err := crypto.SignHash(sigHash2, sk) if err != nil { t.Fatal(err) } txn.TransactionSignatures[0].Signature = sig0[:] txn.TransactionSignatures[1].Signature = sig1[:] txn.TransactionSignatures[2].Signature = sig2[:] // Check that the signing was successful. err = txn.validSignatures(10) if err != nil { t.Error(err) } // Corrupt one of the sigantures. sig0[0]++ txn.TransactionSignatures[0].Signature = sig0[:] err = txn.validSignatures(10) if err == nil { t.Error("Corrupted a signature but the txn was still accepted as valid!") } sig0[0]-- txn.TransactionSignatures[0].Signature = sig0[:] // Fail the validCoveredFields check. txn.TransactionSignatures[0].CoveredFields.SiacoinInputs = []uint64{33} err = txn.validSignatures(10) if err == nil { t.Error("failed to flunk the validCoveredFields check") } txn.TransactionSignatures[0].CoveredFields.SiacoinInputs = nil // Double spend a SiacoinInput, FileContractTermination, and SiafundInput. txn.SiacoinInputs = append(txn.SiacoinInputs, SiacoinInput{UnlockConditions: UnlockConditions{}}) err = txn.validSignatures(10) if err == nil { t.Error("failed to double spend a siacoin input") } txn.SiacoinInputs = txn.SiacoinInputs[:len(txn.SiacoinInputs)-1] txn.FileContractRevisions = append(txn.FileContractRevisions, FileContractRevision{UnlockConditions: UnlockConditions{}}) err = txn.validSignatures(10) if err == nil { t.Error("failed to double spend a file contract termination") } txn.FileContractRevisions = txn.FileContractRevisions[:len(txn.FileContractRevisions)-1] txn.SiafundInputs = append(txn.SiafundInputs, SiafundInput{UnlockConditions: UnlockConditions{}}) err = txn.validSignatures(10) if err == nil { t.Error("failed to double spend a siafund input") } txn.SiafundInputs = txn.SiafundInputs[:len(txn.SiafundInputs)-1] // Add a frivilous signature txn.TransactionSignatures = append(txn.TransactionSignatures, TransactionSignature{}) err = txn.validSignatures(10) if err != ErrFrivilousSignature { t.Error(err) } txn.TransactionSignatures = txn.TransactionSignatures[:len(txn.TransactionSignatures)-1] // Replace one of the cryptography signatures with an always-accepted // signature. This should get rejected because the always-accepted // signature has already been used. tmpTxn0 := txn.TransactionSignatures[0] txn.TransactionSignatures[0] = TransactionSignature{PublicKeyIndex: 1} err = txn.validSignatures(10) if err != ErrPublicKeyOveruse { t.Error(err) } txn.TransactionSignatures[0] = tmpTxn0 // Fail the timelock check for signatures. err = txn.validSignatures(4) if err != ErrPrematureSignature { t.Error(err) } // Try to spend an entropy signature. txn.TransactionSignatures[0] = TransactionSignature{PublicKeyIndex: 2} err = txn.validSignatures(10) if err != ErrEntropyKey { t.Error(err) } txn.TransactionSignatures[0] = tmpTxn0 // Try to point to a nonexistent public key. txn.TransactionSignatures[0] = TransactionSignature{PublicKeyIndex: 5} err = txn.validSignatures(10) if err != ErrInvalidPubKeyIndex { t.Error(err) } txn.TransactionSignatures[0] = tmpTxn0 // Insert a malformed public key into the transaction. txn.SiacoinInputs[0].UnlockConditions.PublicKeys[0].Key = []byte{'b', 'a', 'd'} err = txn.validSignatures(10) if err == nil { t.Error(err) } txn.SiacoinInputs[0].UnlockConditions.PublicKeys[0].Key = pk[:] // Insert a malformed signature into the transaction. txn.TransactionSignatures[0].Signature = []byte{'m', 'a', 'l'} err = txn.validSignatures(10) if err == nil { t.Error(err) } txn.TransactionSignatures[0] = tmpTxn0 // Try to spend a transaction when not every required signature is // available. txn.TransactionSignatures = txn.TransactionSignatures[1:] err = txn.validSignatures(10) if err != ErrMissingSignatures { t.Error(err) } } // TestSiaPublicKeyString does a quick check to verify that the String method // on the SiaPublicKey is producing the expected output. func TestSiaPublicKeyString(t *testing.T) { spk := SiaPublicKey{ Algorithm: SignatureEd25519, Key: make([]byte, 32), } if spk.String() != "ed25519:0000000000000000000000000000000000000000000000000000000000000000" { t.Error("got wrong value for spk.String():", spk.String()) } } Sia-1.0.4/types/target.go000066400000000000000000000063101302033070500152020ustar00rootroot00000000000000package types // target.go defines the target type and implements a few helper functions for // manipulating the target type. import ( "errors" "math/big" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/crypto" ) type ( // A Target is a hash that a block's ID must be "less than" in order for // the block to be considered valid. Miners vary the block's 'Nonce' field // in order to brute-force such an ID. The inverse of a Target is called // the "difficulty," because it is proportional to the amount of time // required to brute-force the Target. Target crypto.Hash ) var ( ErrNegativeTarget = errors.New("negative value used when converting to target") ) // AddDifficulties returns the resulting target with the difficulty of 'x' and // 'y' are added together. Note that the difficulty is the inverse of the // target. The sum is defined by: // sum(x, y) = 1/(1/x + 1/y) func (x Target) AddDifficulties(y Target) (t Target) { sumDifficulty := new(big.Rat).Add(x.Inverse(), y.Inverse()) return RatToTarget(new(big.Rat).Inv(sumDifficulty)) } // Cmp compares the difficulties of two targets. Note that the difficulty is // the inverse of the target. The results are as follows: // -1 if x < y // 0 if x == y // +1 if x > y func (x Target) Cmp(y Target) int { return x.Int().Cmp(y.Int()) } // Difficulty returns the difficulty associated with a given target. func (t Target) Difficulty() Currency { if t == (Target{}) { return NewCurrency(RootDepth.Int()) } return NewCurrency(new(big.Int).Div(RootDepth.Int(), t.Int())) } // Int converts a Target to a big.Int. func (t Target) Int() *big.Int { return new(big.Int).SetBytes(t[:]) } // IntToTarget converts a big.Int to a Target. Negative inputs trigger a panic. func IntToTarget(i *big.Int) (t Target) { // Check for negatives. if i.Sign() < 0 { if build.DEBUG { panic(ErrNegativeTarget) } } else { // In the event of overflow, return the maximum. if i.BitLen() > 256 { return RootDepth } b := i.Bytes() offset := len(t[:]) - len(b) copy(t[offset:], b) } return } // Inverse returns the inverse of a Target as a big.Rat func (t Target) Inverse() *big.Rat { return new(big.Rat).Inv(t.Rat()) } // Mul multiplies the difficulty of a target by y. The product is defined by: // y / x func (x Target) MulDifficulty(y *big.Rat) (t Target) { product := new(big.Rat).Mul(y, x.Inverse()) product = product.Inv(product) return RatToTarget(product) } // Rat converts a Target to a big.Rat. func (t Target) Rat() *big.Rat { return new(big.Rat).SetInt(t.Int()) } // RatToTarget converts a big.Rat to a Target. func RatToTarget(r *big.Rat) (t Target) { if r.Num().Sign() < 0 { if build.DEBUG { panic(ErrNegativeTarget) } } else { i := new(big.Int).Div(r.Num(), r.Denom()) t = IntToTarget(i) } return } // SubtractDifficulties returns the resulting target with the difficulty of 'x' // is subtracted from the target with difficulty 'y'. Note that the difficulty // is the inverse of the target. The difference is defined by: // sum(x, y) = 1/(1/x - 1/y) func (x Target) SubtractDifficulties(y Target) (t Target) { sumDifficulty := new(big.Rat).Sub(x.Inverse(), y.Inverse()) return RatToTarget(new(big.Rat).Inv(sumDifficulty)) } Sia-1.0.4/types/target_test.go000066400000000000000000000123771302033070500162530ustar00rootroot00000000000000package types import ( "math/big" "testing" "github.com/NebulousLabs/Sia/crypto" ) // TestTargetAdd probes the Add function of the target type. func TestTargetAdd(t *testing.T) { var target3, target5, target10 Target target3[crypto.HashSize-1] = 3 target5[crypto.HashSize-1] = 5 target10[crypto.HashSize-1] = 10 expect5 := target10.AddDifficulties(target10) if expect5 != target5 { t.Error("Target.Add not working as expected") } expect3 := target10.AddDifficulties(target5) if expect3 != target3 { t.Error("Target.Add not working as expected") } } // TestTargetCmp probes the Cmp function of the target type. func TestTargetCmp(t *testing.T) { var target1, target2 Target target1[crypto.HashSize-1] = 1 target2[crypto.HashSize-1] = 2 if target1.Cmp(target2) != -1 { t.Error("Target.Cmp not behaving as expected") } if target2.Cmp(target2) != 0 { t.Error("Target.Cmp not behaving as expected") } if target2.Cmp(target1) != 1 { t.Error("Target.Cmp not behaving as expected") } } // TestTargetDifficulty probes the Difficulty function of the target type. func TestTargetDifficulty(t *testing.T) { var target1, target2, target3 Target target2[crypto.HashSize-1] = 1 target3[crypto.HashSize-1] = 2 expDifficulty1 := NewCurrency(RootDepth.Int()) expDifficulty2 := NewCurrency(RootDepth.Int()) expDifficulty3 := NewCurrency(RootDepth.Int()).Div(NewCurrency64(2)) if difficulty := target1.Difficulty(); difficulty.Cmp(expDifficulty1) != 0 { t.Errorf("Expected difficulty %v, got %v", expDifficulty1, difficulty) } if difficulty := target2.Difficulty(); difficulty.Cmp(expDifficulty2) != 0 { t.Errorf("Expected difficulty %v, got %v", expDifficulty2, difficulty) } if difficulty := target3.Difficulty(); difficulty.Cmp(expDifficulty3) != 0 { t.Errorf("Expected difficulty %v, got %v", expDifficulty3, difficulty) } } // TestTargetInt probes the Int function of the target type. func TestTargetInt(t *testing.T) { var target Target target[crypto.HashSize-1] = 1 b := target.Int() if b.Cmp(big.NewInt(1)) != 0 { t.Error("Target.Int did not work correctly") } } // TestTargetIntToTarget probes the IntToTarget function of the target type. func TestTargetIntToTarget(t *testing.T) { var target Target target[crypto.HashSize-1] = 5 b := big.NewInt(5) if IntToTarget(b) != target { t.Error("IntToTarget not working as expected") } } // TestTargetInverse probes the Inverse function of the target type. func TestTargetInverse(t *testing.T) { var target Target target[crypto.HashSize-1] = 2 r := target.Inverse() if r.Num().Cmp(big.NewInt(1)) != 0 { t.Error("Target.Rat did not work as expected") } if r.Denom().Cmp(big.NewInt(2)) != 0 { t.Error("Target.Rat did not work as expected") } } // TestTargetMul probes the Mul function of the target type. func TestTargetMul(t *testing.T) { var target2, target6, target10, target14, target20 Target target2[crypto.HashSize-1] = 2 target6[crypto.HashSize-1] = 6 target10[crypto.HashSize-1] = 10 target14[crypto.HashSize-1] = 14 target20[crypto.HashSize-1] = 20 // Multiplying the difficulty of a target at '10' by 5 will yield a target // of '2'. Similar math follows for the remaining checks. expect2 := target10.MulDifficulty(big.NewRat(5, 1)) if expect2 != target2 { t.Error(expect2) t.Error(target2) t.Error("Target.Mul did not work as expected") } expect6 := target10.MulDifficulty(big.NewRat(3, 2)) if expect6 != target6 { t.Error("Target.Mul did not work as expected") } expect14 := target10.MulDifficulty(big.NewRat(7, 10)) if expect14 != target14 { t.Error("Target.Mul did not work as expected") } expect20 := target10.MulDifficulty(big.NewRat(1, 2)) if expect20 != target20 { t.Error("Target.Mul did not work as expected") } } // TestTargetRat probes the Rat function of the target type. func TestTargetRat(t *testing.T) { var target Target target[crypto.HashSize-1] = 3 r := target.Rat() if r.Num().Cmp(big.NewInt(3)) != 0 { t.Error("Target.Rat did not work as expected") } if r.Denom().Cmp(big.NewInt(1)) != 0 { t.Error("Target.Rat did not work as expected") } } // TestTargetOverflow checks that IntToTarget will return a maximum target if // there is an overflow. func TestTargetOverflow(t *testing.T) { largeInt := new(big.Int).Lsh(big.NewInt(1), 260) expectRoot := IntToTarget(largeInt) if expectRoot != RootDepth { t.Error("IntToTarget does not properly handle overflows") } } // TestTargetNegativeIntToTarget tries to create a negative target using // IntToTarget. func TestTargetNegativeIntToTarget(t *testing.T) { if testing.Short() { t.SkipNow() } // In debug mode, attempting to create a negative target should trigger a // panic. defer func() { r := recover() if r != ErrNegativeTarget { t.Error("no panic occurred when trying to create a negative target") } }() b := big.NewInt(-3) _ = IntToTarget(b) } // TestTargetNegativeRatToTarget tries to create a negative target using // RatToTarget. func TestTargetNegativeRatToTarget(t *testing.T) { if testing.Short() { t.SkipNow() } // In debug mode, attempting to create a negative target should trigger a // panic. defer func() { r := recover() if r != ErrNegativeTarget { t.Error("no panic occurred when trying to create a negative target") } }() r := big.NewRat(3, -5) _ = RatToTarget(r) } Sia-1.0.4/types/timestamp.go000066400000000000000000000017311302033070500157210ustar00rootroot00000000000000package types // timestamp.go defines the timestamp type and implements sort.Interface // interface for slices of timestamps. import ( "time" ) type ( Timestamp uint64 TimestampSlice []Timestamp ) // CurrentTimestamp returns the current time as a Timestamp. func CurrentTimestamp() Timestamp { return Timestamp(time.Now().Unix()) } // Len is part of sort.Interface func (ts TimestampSlice) Len() int { return len(ts) } // Less is part of sort.Interface func (ts TimestampSlice) Less(i, j int) bool { return ts[i] < ts[j] } // Swap is part of sort.Interface func (ts TimestampSlice) Swap(i, j int) { ts[i], ts[j] = ts[j], ts[i] } // Clock allows clients to retrieve the current time. type Clock interface { Now() Timestamp } // StdClock is an implementation of Clock that retrieves the current time using // the system time. type StdClock struct{} // Now retrieves the current timestamp. func (c StdClock) Now() Timestamp { return Timestamp(time.Now().Unix()) } Sia-1.0.4/types/timestamp_test.go000066400000000000000000000010341302033070500167540ustar00rootroot00000000000000package types import ( "sort" "testing" ) // TestTimestampSorting verifies that using sort.Sort accurately sorts // timestamps. func TestTimestampSorting(t *testing.T) { ts := TimestampSlice{ CurrentTimestamp(), CurrentTimestamp() - 5, CurrentTimestamp() + 5, CurrentTimestamp() + 12, CurrentTimestamp() - 3, CurrentTimestamp() - 25, } sort.Sort(ts) currentTime := ts[0] for _, timestamp := range ts { if timestamp < currentTime { t.Error("timestamp slice not properly sorted") } currentTime = timestamp } } Sia-1.0.4/types/transactions.go000066400000000000000000000274141302033070500164340ustar00rootroot00000000000000package types // transaction.go defines the transaction type and all of the sub-fields of the // transaction, as well as providing helper functions for working with // transactions. The various IDs are designed such that, in a legal blockchain, // it is cryptographically unlikely that any two objects would share an id. import ( "encoding/json" "errors" "fmt" "github.com/NebulousLabs/Sia/crypto" ) const ( SpecifierLen = 16 ) // These Specifiers are used internally when calculating a type's ID. See // Specifier for more details. var ( SpecifierMinerPayout = Specifier{'m', 'i', 'n', 'e', 'r', ' ', 'p', 'a', 'y', 'o', 'u', 't'} SpecifierSiacoinInput = Specifier{'s', 'i', 'a', 'c', 'o', 'i', 'n', ' ', 'i', 'n', 'p', 'u', 't'} SpecifierSiacoinOutput = Specifier{'s', 'i', 'a', 'c', 'o', 'i', 'n', ' ', 'o', 'u', 't', 'p', 'u', 't'} SpecifierFileContract = Specifier{'f', 'i', 'l', 'e', ' ', 'c', 'o', 'n', 't', 'r', 'a', 'c', 't'} SpecifierFileContractRevision = Specifier{'f', 'i', 'l', 'e', ' ', 'c', 'o', 'n', 't', 'r', 'a', 'c', 't', ' ', 'r', 'e'} SpecifierStorageProof = Specifier{'s', 't', 'o', 'r', 'a', 'g', 'e', ' ', 'p', 'r', 'o', 'o', 'f'} SpecifierStorageProofOutput = Specifier{'s', 't', 'o', 'r', 'a', 'g', 'e', ' ', 'p', 'r', 'o', 'o', 'f'} SpecifierSiafundInput = Specifier{'s', 'i', 'a', 'f', 'u', 'n', 'd', ' ', 'i', 'n', 'p', 'u', 't'} SpecifierSiafundOutput = Specifier{'s', 'i', 'a', 'f', 'u', 'n', 'd', ' ', 'o', 'u', 't', 'p', 'u', 't'} SpecifierClaimOutput = Specifier{'c', 'l', 'a', 'i', 'm', ' ', 'o', 'u', 't', 'p', 'u', 't'} SpecifierMinerFee = Specifier{'m', 'i', 'n', 'e', 'r', ' ', 'f', 'e', 'e'} ErrTransactionIDWrongLen = errors.New("input has wrong length to be an encoded transaction id") ) type ( // A Specifier is a fixed-length byte-array that serves two purposes. In // the wire protocol, they are used to identify a particular encoding // algorithm, signature algorithm, etc. This allows nodes to communicate on // their own terms; for example, to reduce bandwidth costs, a node might // only accept compressed messages. // // Internally, Specifiers are used to guarantee unique IDs. Various // consensus types have an associated ID, calculated by hashing the data // contained in the type. By prepending the data with Specifier, we can // guarantee that distinct types will never produce the same hash. Specifier [SpecifierLen]byte // IDs are used to refer to a type without revealing its contents. They // are constructed by hashing specific fields of the type, along with a // Specifier. While all of these types are hashes, defining type aliases // gives us type safety and makes the code more readable. TransactionID crypto.Hash SiacoinOutputID crypto.Hash SiafundOutputID crypto.Hash FileContractID crypto.Hash OutputID crypto.Hash // A Transaction is an atomic component of a block. Transactions can contain // inputs and outputs, file contracts, storage proofs, and even arbitrary // data. They can also contain signatures to prove that a given party has // approved the transaction, or at least a particular subset of it. // // Transactions can depend on other previous transactions in the same block, // but transactions cannot spend outputs that they create or otherwise be // self-dependent. Transaction struct { SiacoinInputs []SiacoinInput `json:"siacoininputs"` SiacoinOutputs []SiacoinOutput `json:"siacoinoutputs"` FileContracts []FileContract `json:"filecontracts"` FileContractRevisions []FileContractRevision `json:"filecontractrevisions"` StorageProofs []StorageProof `json:"storageproofs"` SiafundInputs []SiafundInput `json:"siafundinputs"` SiafundOutputs []SiafundOutput `json:"siafundoutputs"` MinerFees []Currency `json:"minerfees"` ArbitraryData [][]byte `json:"arbitrarydata"` TransactionSignatures []TransactionSignature `json:"transactionsignatures"` } // A SiacoinInput consumes a SiacoinOutput and adds the siacoins to the set of // siacoins that can be spent in the transaction. The ParentID points to the // output that is getting consumed, and the UnlockConditions contain the rules // for spending the output. The UnlockConditions must match the UnlockHash of // the output. SiacoinInput struct { ParentID SiacoinOutputID `json:"parentid"` UnlockConditions UnlockConditions `json:"unlockconditions"` } // A SiacoinOutput holds a volume of siacoins. Outputs must be spent // atomically; that is, they must all be spent in the same transaction. The // UnlockHash is the hash of the UnlockConditions that must be fulfilled // in order to spend the output. SiacoinOutput struct { Value Currency `json:"value"` UnlockHash UnlockHash `json:"unlockhash"` } // A SiafundInput consumes a SiafundOutput and adds the siafunds to the set of // siafunds that can be spent in the transaction. The ParentID points to the // output that is getting consumed, and the UnlockConditions contain the rules // for spending the output. The UnlockConditions must match the UnlockHash of // the output. SiafundInput struct { ParentID SiafundOutputID `json:"parentid"` UnlockConditions UnlockConditions `json:"unlockconditions"` ClaimUnlockHash UnlockHash `json:"claimunlockhash"` } // A SiafundOutput holds a volume of siafunds. Outputs must be spent // atomically; that is, they must all be spent in the same transaction. The // UnlockHash is the hash of a set of UnlockConditions that must be fulfilled // in order to spend the output. // // When the SiafundOutput is spent, a SiacoinOutput is created, where: // // SiacoinOutput.Value := (SiafundPool - ClaimStart) / 10,000 // SiacoinOutput.UnlockHash := SiafundOutput.ClaimUnlockHash // // When a SiafundOutput is put into a transaction, the ClaimStart must always // equal zero. While the transaction is being processed, the ClaimStart is set // to the value of the SiafundPool. SiafundOutput struct { Value Currency `json:"value"` UnlockHash UnlockHash `json:"unlockhash"` ClaimStart Currency `json:"claimstart"` } ) // ID returns the id of a transaction, which is taken by marshalling all of the // fields except for the signatures and taking the hash of the result. func (t Transaction) ID() TransactionID { return TransactionID(crypto.HashAll( t.SiacoinInputs, t.SiacoinOutputs, t.FileContracts, t.FileContractRevisions, t.StorageProofs, t.SiafundInputs, t.SiafundOutputs, t.MinerFees, t.ArbitraryData, )) } // SiacoinOutputID returns the ID of a siacoin output at the given index, // which is calculated by hashing the concatenation of the SiacoinOutput // Specifier, all of the fields in the transaction (except the signatures), // and output index. func (t Transaction) SiacoinOutputID(i uint64) SiacoinOutputID { return SiacoinOutputID(crypto.HashAll( SpecifierSiacoinOutput, t.SiacoinInputs, t.SiacoinOutputs, t.FileContracts, t.FileContractRevisions, t.StorageProofs, t.SiafundInputs, t.SiafundOutputs, t.MinerFees, t.ArbitraryData, i, )) } // FileContractID returns the ID of a file contract at the given index, which // is calculated by hashing the concatenation of the FileContract Specifier, // all of the fields in the transaction (except the signatures), and the // contract index. func (t Transaction) FileContractID(i uint64) FileContractID { return FileContractID(crypto.HashAll( SpecifierFileContract, t.SiacoinInputs, t.SiacoinOutputs, t.FileContracts, t.FileContractRevisions, t.StorageProofs, t.SiafundInputs, t.SiafundOutputs, t.MinerFees, t.ArbitraryData, i, )) } // SiafundOutputID returns the ID of a SiafundOutput at the given index, which // is calculated by hashing the concatenation of the SiafundOutput Specifier, // all of the fields in the transaction (except the signatures), and output // index. func (t Transaction) SiafundOutputID(i uint64) SiafundOutputID { return SiafundOutputID(crypto.HashAll( SpecifierSiafundOutput, t.SiacoinInputs, t.SiacoinOutputs, t.FileContracts, t.FileContractRevisions, t.StorageProofs, t.SiafundInputs, t.SiafundOutputs, t.MinerFees, t.ArbitraryData, i, )) } // SiacoinOutputSum returns the sum of all the siacoin outputs in the // transaction, which must match the sum of all the siacoin inputs. Siacoin // outputs created by storage proofs and siafund outputs are not considered, as // they were considered when the contract responsible for funding them was // created. func (t Transaction) SiacoinOutputSum() (sum Currency) { // Add the siacoin outputs. for _, sco := range t.SiacoinOutputs { sum = sum.Add(sco.Value) } // Add the file contract payouts. for _, fc := range t.FileContracts { sum = sum.Add(fc.Payout) } // Add the miner fees. for _, fee := range t.MinerFees { sum = sum.Add(fee) } return } // SiaClaimOutputID returns the ID of the SiacoinOutput that is created when // the siafund output is spent. The ID is the hash the SiafundOutputID. func (id SiafundOutputID) SiaClaimOutputID() SiacoinOutputID { return SiacoinOutputID(crypto.HashObject(id)) } // MarshalJSON marshals a specifier as a string. func (s Specifier) MarshalJSON() ([]byte, error) { return json.Marshal(s.String()) } // String returns the specifier as a string, trimming any trailing zeros. func (s Specifier) String() string { var i int for i = range s { if s[i] == 0 { break } } return string(s[:i]) } // UnmarshalJSON decodes the json string of the specifier. func (s *Specifier) UnmarshalJSON(b []byte) error { var str string if err := json.Unmarshal(b, &str); err != nil { return err } copy(s[:], str) return nil } // MarshalJSON marshals an id as a hex string. func (tid TransactionID) MarshalJSON() ([]byte, error) { return json.Marshal(tid.String()) } // String prints the id in hex. func (tid TransactionID) String() string { return fmt.Sprintf("%x", tid[:]) } // UnmarshalJSON decodes the json hex string of the id. func (tid *TransactionID) UnmarshalJSON(b []byte) error { return (*crypto.Hash)(tid).UnmarshalJSON(b) } // MarshalJSON marshals an id as a hex string. func (oid OutputID) MarshalJSON() ([]byte, error) { return json.Marshal(oid.String()) } // String prints the id in hex. func (oid OutputID) String() string { return fmt.Sprintf("%x", oid[:]) } // UnmarshalJSON decodes the json hex string of the id. func (oid *OutputID) UnmarshalJSON(b []byte) error { return (*crypto.Hash)(oid).UnmarshalJSON(b) } // MarshalJSON marshals an id as a hex string. func (scoid SiacoinOutputID) MarshalJSON() ([]byte, error) { return json.Marshal(scoid.String()) } // String prints the id in hex. func (scoid SiacoinOutputID) String() string { return fmt.Sprintf("%x", scoid[:]) } // UnmarshalJSON decodes the json hex string of the id. func (scoid *SiacoinOutputID) UnmarshalJSON(b []byte) error { return (*crypto.Hash)(scoid).UnmarshalJSON(b) } // MarshalJSON marshals an id as a hex string. func (fcid FileContractID) MarshalJSON() ([]byte, error) { return json.Marshal(fcid.String()) } // String prints the id in hex. func (fcid FileContractID) String() string { return fmt.Sprintf("%x", fcid[:]) } // UnmarshalJSON decodes the json hex string of the id. func (fcid *FileContractID) UnmarshalJSON(b []byte) error { return (*crypto.Hash)(fcid).UnmarshalJSON(b) } // MarshalJSON marshals an id as a hex string. func (sfoid SiafundOutputID) MarshalJSON() ([]byte, error) { return json.Marshal(sfoid.String()) } // String prints the id in hex. func (sfoid SiafundOutputID) String() string { return fmt.Sprintf("%x", sfoid[:]) } // UnmarshalJSON decodes the json hex string of the id. func (sfoid *SiafundOutputID) UnmarshalJSON(b []byte) error { return (*crypto.Hash)(sfoid).UnmarshalJSON(b) } Sia-1.0.4/types/transactions_test.go000066400000000000000000000043351302033070500174700ustar00rootroot00000000000000package types import ( "encoding/json" "testing" "github.com/NebulousLabs/Sia/crypto" ) // TestTransactionIDs probes all of the ID functions of the Transaction type. func TestIDs(t *testing.T) { // Create every type of ID using empty fields. txn := Transaction{ SiacoinOutputs: []SiacoinOutput{{}}, FileContracts: []FileContract{{}}, SiafundOutputs: []SiafundOutput{{}}, } tid := txn.ID() scoid := txn.SiacoinOutputID(0) fcid := txn.FileContractID(0) spidT := fcid.StorageProofOutputID(ProofValid, 0) spidF := fcid.StorageProofOutputID(ProofMissed, 0) sfoid := txn.SiafundOutputID(0) scloid := sfoid.SiaClaimOutputID() // Put all of the ids into a slice. var ids []crypto.Hash ids = append(ids, crypto.Hash(tid), crypto.Hash(scoid), crypto.Hash(fcid), crypto.Hash(spidT), crypto.Hash(spidF), crypto.Hash(sfoid), crypto.Hash(scloid), ) // Check that each id is unique. knownIDs := make(map[crypto.Hash]struct{}) for i, id := range ids { _, exists := knownIDs[id] if exists { t.Error("id repeat for index", i) } knownIDs[id] = struct{}{} } } // TestTransactionSiacoinOutputSum probes the SiacoinOutputSum method of the // Transaction type. func TestTransactionSiacoinOutputSum(t *testing.T) { // Create a transaction with all types of siacoin outputs. txn := Transaction{ SiacoinOutputs: []SiacoinOutput{ {Value: NewCurrency64(1)}, {Value: NewCurrency64(20)}, }, FileContracts: []FileContract{ {Payout: NewCurrency64(300)}, {Payout: NewCurrency64(4000)}, }, MinerFees: []Currency{ NewCurrency64(50000), NewCurrency64(600000), }, } if txn.SiacoinOutputSum().Cmp(NewCurrency64(654321)) != 0 { t.Error("wrong siacoin output sum was calculated, got:", txn.SiacoinOutputSum()) } } // TestSpecifierMarshaling tests the marshaling methods of the specifier // type. func TestSpecifierMarshaling(t *testing.T) { s1 := SpecifierClaimOutput b, err := json.Marshal(s1) if err != nil { t.Fatal(err) } var s2 Specifier err = json.Unmarshal(b, &s2) if err != nil { t.Fatal(err) } else if s2 != s1 { t.Fatal("mismatch:", s1, s2) } // invalid json x := 3 b, _ = json.Marshal(x) err = json.Unmarshal(b, &s2) if err == nil { t.Fatal("Unmarshal should have failed") } } Sia-1.0.4/types/unlockhash.go000066400000000000000000000057641302033070500160670ustar00rootroot00000000000000package types import ( "bytes" "encoding/json" "fmt" "github.com/NebulousLabs/Sia/crypto" ) // unlockhash.go contains the unlockhash alias along with usability methods // such as String and an implementation of sort.Interface. const ( // UnlockHashChecksumSize is the size of the checksum used to verify // human-readable addresses. It is not a crypytographically secure // checksum, it's merely intended to prevent typos. 6 is chosen because it // brings the total size of the address to 38 bytes, leaving 2 bytes for // potential version additions in the future. UnlockHashChecksumSize = 6 ) // An UnlockHash is a specially constructed hash of the UnlockConditions type. // "Locked" values can be unlocked by providing the UnlockConditions that hash // to a given UnlockHash. See SpendConditions.UnlockHash for details on how the // UnlockHash is constructed. type UnlockHash crypto.Hash type UnlockHashSlice []UnlockHash // MarshalJSON is implemented on the unlock hash to always produce a hex string // upon marshalling. func (uh UnlockHash) MarshalJSON() ([]byte, error) { return json.Marshal(uh.String()) } // UnmarshalJSON is implemented on the unlock hash to recover an unlock hash // that has been encoded to a hex string. func (uh *UnlockHash) UnmarshalJSON(b []byte) error { // Check the length of b. if len(b) != crypto.HashSize*2+UnlockHashChecksumSize*2+2 && len(b) != crypto.HashSize*2+2 { return ErrUnlockHashWrongLen } return uh.LoadString(string(b[1 : len(b)-1])) } // String returns the hex representation of the unlock hash as a string - this // includes a checksum. func (uh UnlockHash) String() string { uhChecksum := crypto.HashObject(uh) return fmt.Sprintf("%x%x", uh[:], uhChecksum[:UnlockHashChecksumSize]) } // LoadString loads a hex representation (including checksum) of an unlock hash // into an unlock hash object. An error is returned if the string is invalid or // fails the checksum. func (uh *UnlockHash) LoadString(strUH string) error { // Check the length of strUH. if len(strUH) != crypto.HashSize*2+UnlockHashChecksumSize*2 { return ErrUnlockHashWrongLen } // Decode the unlock hash. var byteUnlockHash []byte var checksum []byte _, err := fmt.Sscanf(strUH[:crypto.HashSize*2], "%x", &byteUnlockHash) if err != nil { return err } // Decode and verify the checksum. _, err = fmt.Sscanf(strUH[crypto.HashSize*2:], "%x", &checksum) if err != nil { return err } expectedChecksum := crypto.HashBytes(byteUnlockHash) if !bytes.Equal(expectedChecksum[:UnlockHashChecksumSize], checksum) { return ErrInvalidUnlockHashChecksum } copy(uh[:], byteUnlockHash[:]) return nil } // Len implements the Len method of sort.Interface. func (uhs UnlockHashSlice) Len() int { return len(uhs) } // Less implements the Less method of sort.Interface. func (uhs UnlockHashSlice) Less(i, j int) bool { return bytes.Compare(uhs[i][:], uhs[j][:]) < 0 } // Swap implements the Swap method of sort.Interface. func (uhs UnlockHashSlice) Swap(i, j int) { uhs[i], uhs[j] = uhs[j], uhs[i] } Sia-1.0.4/types/unlockhash_test.go000066400000000000000000000057371302033070500171260ustar00rootroot00000000000000package types import ( "encoding/json" "sort" "testing" ) // TestUnlockHashJSONMarshalling checks that when an unlock hash is marshalled // and unmarshalled using JSON, the result is what is expected. func TestUnlockHashJSONMarshalling(t *testing.T) { // Create an unlock hash. uc := UnlockConditions{ Timelock: 5, SignaturesRequired: 3, } uh := uc.UnlockHash() // Marshal the unlock hash. marUH, err := json.Marshal(uh) if err != nil { t.Fatal(err) } // Unmarshal the unlock hash and compare to the original. var umarUH UnlockHash err = json.Unmarshal(marUH, &umarUH) if err != nil { t.Fatal(err) } if umarUH != uh { t.Error("Marshalled and unmarshalled unlock hash are not equivalent") } // Corrupt the checksum. marUH[36]++ err = umarUH.UnmarshalJSON(marUH) if err != ErrInvalidUnlockHashChecksum { t.Error("expecting an invalid checksum:", err) } marUH[36]-- // Try an input that's not correct hex. marUH[7] += 100 err = umarUH.UnmarshalJSON(marUH) if err == nil { t.Error("Expecting error after corrupting input") } marUH[7] -= 100 // Try an input of the wrong length. err = (&umarUH).UnmarshalJSON(marUH[2:]) if err != ErrUnlockHashWrongLen { t.Error("Got wrong error:", err) } } // TestUnlockHashStringMarshalling checks that when an unlock hash is // marshalled and unmarshalled using String and LoadString, the result is what // is expected. func TestUnlockHashStringMarshalling(t *testing.T) { // Create an unlock hash. uc := UnlockConditions{ Timelock: 2, SignaturesRequired: 7, } uh := uc.UnlockHash() // Marshal the unlock hash. marUH := uh.String() // Unmarshal the unlock hash and compare to the original. var umarUH UnlockHash err := umarUH.LoadString(marUH) if err != nil { t.Fatal(err) } if umarUH != uh { t.Error("Marshalled and unmarshalled unlock hash are not equivalent") } // Corrupt the checksum. byteMarUH := []byte(marUH) byteMarUH[36]++ err = umarUH.LoadString(string(byteMarUH)) if err != ErrInvalidUnlockHashChecksum { t.Error("expecting an invalid checksum:", err) } byteMarUH[36]-- // Try an input that's not correct hex. byteMarUH[7] += 100 err = umarUH.LoadString(string(byteMarUH)) if err == nil { t.Error("Expecting error after corrupting input") } byteMarUH[7] -= 100 // Try an input of the wrong length. err = umarUH.LoadString(string(byteMarUH[2:])) if err != ErrUnlockHashWrongLen { t.Error("Got wrong error:", err) } } // TestUnlockHashSliceSorting checks that the sort method correctly sorts // unlock hash slices. func TestUnlockHashSliceSorting(t *testing.T) { // To test that byte-order is done correctly, a semi-random second byte is // used that is equal to the first byte * 23 % 7 uhs := UnlockHashSlice{ UnlockHash{4, 1}, UnlockHash{0, 0}, UnlockHash{2, 4}, UnlockHash{3, 6}, UnlockHash{1, 2}, } sort.Sort(uhs) for i := byte(0); i < 5; i++ { if uhs[i] != (UnlockHash{i, (i * 23) % 7}) { t.Error("sorting failed on index", i, uhs[i]) } } } Sia-1.0.4/types/validtransaction.go000066400000000000000000000233661302033070500172730ustar00rootroot00000000000000package types // validtransaction.go has functions for checking whether a transaction is // valid outside of the context of a consensus set. This means checking the // size of the transaction, the content of the signatures, and a large set of // other rules that are inherent to how a transaction should be constructed. import ( "errors" "github.com/NebulousLabs/Sia/encoding" ) var ( ErrDoubleSpend = errors.New("transaction uses a parent object twice") ErrFileContractWindowEndViolation = errors.New("file contract window must end at least one block after it starts") ErrFileContractWindowStartViolation = errors.New("file contract window must start in the future") ErrFileContractOutputSumViolation = errors.New("file contract has invalid output sums") ErrNonZeroClaimStart = errors.New("transaction has a siafund output with a non-zero siafund claim") ErrNonZeroRevision = errors.New("new file contract has a nonzero revision number") ErrStorageProofWithOutputs = errors.New("transaction has both a storage proof and other outputs") ErrTimelockNotSatisfied = errors.New("timelock has not been met") ErrTransactionTooLarge = errors.New("transaction is too large to fit in a block") ErrZeroMinerFee = errors.New("transaction has a zero value miner fee") ErrZeroOutput = errors.New("transaction cannot have an output or payout that has zero value") ErrZeroRevision = errors.New("transaction has a file contract revision with RevisionNumber=0") ) // correctFileContracts checks that the file contracts adhere to the file // contract rules. func (t Transaction) correctFileContracts(currentHeight BlockHeight) error { // Check that FileContract rules are being followed. for _, fc := range t.FileContracts { // Check that start and expiration are reasonable values. if fc.WindowStart <= currentHeight { return ErrFileContractWindowStartViolation } if fc.WindowEnd <= fc.WindowStart { return ErrFileContractWindowEndViolation } // Check that the proof outputs sum to the payout after the // siafund fee has been applied. var validProofOutputSum, missedProofOutputSum Currency for _, output := range fc.ValidProofOutputs { /* - Future hardforking code. if output.Value.IsZero() { return ErrZeroOutput } */ validProofOutputSum = validProofOutputSum.Add(output.Value) } for _, output := range fc.MissedProofOutputs { /* - Future hardforking code. if output.Value.IsZero() { return ErrZeroOutput } */ missedProofOutputSum = missedProofOutputSum.Add(output.Value) } outputPortion := PostTax(currentHeight, fc.Payout) if validProofOutputSum.Cmp(outputPortion) != 0 { return ErrFileContractOutputSumViolation } if missedProofOutputSum.Cmp(outputPortion) != 0 { return ErrFileContractOutputSumViolation } } return nil } // correctFileContractRevisions checks that any file contract revisions adhere // to the revision rules. func (t Transaction) correctFileContractRevisions(currentHeight BlockHeight) error { for _, fcr := range t.FileContractRevisions { // Check that start and expiration are reasonable values. if fcr.NewWindowStart <= currentHeight { return ErrFileContractWindowStartViolation } if fcr.NewWindowEnd <= fcr.NewWindowStart { return ErrFileContractWindowEndViolation } // Check that the valid outputs and missed outputs sum to the same // value. var validProofOutputSum, missedProofOutputSum Currency for _, output := range fcr.NewValidProofOutputs { /* - Future hardforking code. if output.Value.IsZero() { return ErrZeroOutput } */ validProofOutputSum = validProofOutputSum.Add(output.Value) } for _, output := range fcr.NewMissedProofOutputs { /* - Future hardforking code. if output.Value.IsZero() { return ErrZeroOutput } */ missedProofOutputSum = missedProofOutputSum.Add(output.Value) } if validProofOutputSum.Cmp(missedProofOutputSum) != 0 { return ErrFileContractOutputSumViolation } } return nil } // fitsInABlock checks if the transaction is likely to fit in a block. // Currently there is no limitation on transaction size other than it must fit // in a block. func (t Transaction) fitsInABlock() error { // Check that the transaction will fit inside of a block, leaving 5kb for // overhead. if uint64(len(encoding.Marshal(t))) > BlockSizeLimit-5e3 { return ErrTransactionTooLarge } return nil } // followsMinimumValues checks that all outputs adhere to the rules for the // minimum allowed value (generally 1). func (t Transaction) followsMinimumValues() error { for _, sco := range t.SiacoinOutputs { if sco.Value.IsZero() { return ErrZeroOutput } } for _, fc := range t.FileContracts { if fc.Payout.IsZero() { return ErrZeroOutput } } for _, sfo := range t.SiafundOutputs { // SiafundOutputs are special in that they have a reserved field, the // ClaimStart, which gets sent over the wire but must always be set to // 0. The Value must always be greater than 0. if !sfo.ClaimStart.IsZero() { return ErrNonZeroClaimStart } if sfo.Value.IsZero() { return ErrZeroOutput } } for _, fee := range t.MinerFees { if fee.IsZero() { return ErrZeroMinerFee } } return nil } // FollowsStorageProofRules checks that a transaction follows the limitations // placed on transactions that have storage proofs. func (t Transaction) followsStorageProofRules() error { // No storage proofs, no problems. if len(t.StorageProofs) == 0 { return nil } // If there are storage proofs, there can be no siacoin outputs, siafund // outputs, new file contracts, or file contract terminations. These // restrictions are in place because a storage proof can be invalidated by // a simple reorg, which will also invalidate the rest of the transaction. // These restrictions minimize blockchain turbulence. These other types // cannot be invalidated by a simple reorg, and must instead by replaced by // a conflicting transaction. if len(t.SiacoinOutputs) != 0 { return ErrStorageProofWithOutputs } if len(t.FileContracts) != 0 { return ErrStorageProofWithOutputs } if len(t.FileContractRevisions) != 0 { return ErrStorageProofWithOutputs } if len(t.SiafundOutputs) != 0 { return ErrStorageProofWithOutputs } return nil } // noRepeats checks that a transaction does not spend multiple outputs twice, // submit two valid storage proofs for the same file contract, etc. We // frivilously check that a file contract termination and storage proof don't // act on the same file contract. There is very little overhead for doing so, // and the check is only frivilous because of the current rule that file // contract terminations are not valid after the proof window opens. func (t Transaction) noRepeats() error { // Check that there are no repeat instances of siacoin outputs, storage // proofs, contract terminations, or siafund outputs. siacoinInputs := make(map[SiacoinOutputID]struct{}) for _, sci := range t.SiacoinInputs { _, exists := siacoinInputs[sci.ParentID] if exists { return ErrDoubleSpend } siacoinInputs[sci.ParentID] = struct{}{} } doneFileContracts := make(map[FileContractID]struct{}) for _, sp := range t.StorageProofs { _, exists := doneFileContracts[sp.ParentID] if exists { return ErrDoubleSpend } doneFileContracts[sp.ParentID] = struct{}{} } for _, fcr := range t.FileContractRevisions { _, exists := doneFileContracts[fcr.ParentID] if exists { return ErrDoubleSpend } doneFileContracts[fcr.ParentID] = struct{}{} } siafundInputs := make(map[SiafundOutputID]struct{}) for _, sfi := range t.SiafundInputs { _, exists := siafundInputs[sfi.ParentID] if exists { return ErrDoubleSpend } siafundInputs[sfi.ParentID] = struct{}{} } return nil } // validUnlockConditions checks that the conditions of uc have been met. The // height is taken as input so that modules who might be at a different height // can do the verification without needing to use their own function. // Additionally, it means that the function does not need to be a method of the // consensus set. func validUnlockConditions(uc UnlockConditions, currentHeight BlockHeight) (err error) { if uc.Timelock > currentHeight { return ErrTimelockNotSatisfied } return } // validUnlockConditions checks that all of the unlock conditions in the // transaction are valid. func (t Transaction) validUnlockConditions(currentHeight BlockHeight) (err error) { for _, sci := range t.SiacoinInputs { err = validUnlockConditions(sci.UnlockConditions, currentHeight) if err != nil { return } } for _, fcr := range t.FileContractRevisions { err = validUnlockConditions(fcr.UnlockConditions, currentHeight) if err != nil { return } } for _, sfi := range t.SiafundInputs { err = validUnlockConditions(sfi.UnlockConditions, currentHeight) if err != nil { return } } return } // StandaloneValid returns an error if a transaction is not valid in any // context, for example if the same output is spent twice in the same // transaction. StandaloneValid will not check that all outputs being spent are // legal outputs, as it has no confirmed or unconfirmed set to look at. func (t Transaction) StandaloneValid(currentHeight BlockHeight) (err error) { err = t.fitsInABlock() if err != nil { return } err = t.followsStorageProofRules() if err != nil { return } err = t.noRepeats() if err != nil { return } err = t.followsMinimumValues() if err != nil { return } err = t.correctFileContracts(currentHeight) if err != nil { return } err = t.correctFileContractRevisions(currentHeight) if err != nil { return } err = t.validUnlockConditions(currentHeight) if err != nil { return } err = t.validSignatures(currentHeight) if err != nil { return } return } Sia-1.0.4/types/validtransaction_bench_test.go000066400000000000000000000027631302033070500214670ustar00rootroot00000000000000package types import ( "testing" "github.com/NebulousLabs/Sia/crypto" "github.com/NebulousLabs/Sia/encoding" ) // BenchmarkStandaloneValid times how long it takes to verify a single // large transaction, with a certain number of signatures func BenchmarkStandaloneValid(b *testing.B) { numSigs := 7 // make a transaction numSigs with valid inputs with valid signatures b.ReportAllocs() txn := Transaction{} sk := make([]crypto.SecretKey, numSigs) pk := make([]crypto.PublicKey, numSigs) for i := 0; i < numSigs; i++ { s, p, err := crypto.GenerateKeyPair() if err != nil { b.Fatal(err) } sk[i] = s pk[i] = p uc := UnlockConditions{ PublicKeys: []SiaPublicKey{ {Algorithm: SignatureEd25519, Key: pk[i][:]}, }, SignaturesRequired: 1, } txn.SiacoinInputs = append(txn.SiacoinInputs, SiacoinInput{ UnlockConditions: uc, }) copy(txn.SiacoinInputs[i].ParentID[:], encoding.Marshal(i)) txn.TransactionSignatures = append(txn.TransactionSignatures, TransactionSignature{ CoveredFields: CoveredFields{WholeTransaction: true}, }) copy(txn.TransactionSignatures[i].ParentID[:], encoding.Marshal(i)) } // Transaction must be constructed before signing for i := 0; i < numSigs; i++ { sigHash := txn.SigHash(i) sig0, err := crypto.SignHash(sigHash, sk[i]) if err != nil { b.Fatal(err) } txn.TransactionSignatures[i].Signature = sig0[:] } b.ResetTimer() for i := 0; i < b.N; i++ { err := txn.StandaloneValid(10) if err != nil { b.Fatal(err) } } } Sia-1.0.4/types/validtransaction_test.go000066400000000000000000000316521302033070500203270ustar00rootroot00000000000000package types import ( "testing" ) // TestTransactionCorrectFileContracts probes the correctFileContracts function // of the Transaction type. func TestTransactionCorrectFileContracts(t *testing.T) { // Try a transaction with a FileContract that is correct. txn := Transaction{ FileContracts: []FileContract{ { WindowStart: 35, WindowEnd: 40, Payout: NewCurrency64(1e6), ValidProofOutputs: []SiacoinOutput{ {Value: NewCurrency64(70e3)}, {Value: NewCurrency64(900e3)}, }, MissedProofOutputs: []SiacoinOutput{ {Value: NewCurrency64(70e3)}, {Value: NewCurrency64(900e3)}, }, }, }, } err := txn.correctFileContracts(30) if err != nil { t.Error(err) } // Try when the start height was missed. err = txn.correctFileContracts(35) if err != ErrFileContractWindowStartViolation { t.Error(err) } err = txn.correctFileContracts(135) if err != ErrFileContractWindowStartViolation { t.Error(err) } // Try when the expiration equal to and less than the start. txn.FileContracts[0].WindowEnd = 35 err = txn.correctFileContracts(30) if err != ErrFileContractWindowEndViolation { t.Error(err) } txn.FileContracts[0].WindowEnd = 35 err = txn.correctFileContracts(30) if err != ErrFileContractWindowEndViolation { t.Error(err) } txn.FileContracts[0].WindowEnd = 40 // Attempt under and over output sums. txn.FileContracts[0].ValidProofOutputs[0].Value = NewCurrency64(69e3) err = txn.correctFileContracts(30) if err != ErrFileContractOutputSumViolation { t.Error(err) } txn.FileContracts[0].ValidProofOutputs[0].Value = NewCurrency64(71e3) err = txn.correctFileContracts(30) if err != ErrFileContractOutputSumViolation { t.Error(err) } txn.FileContracts[0].ValidProofOutputs[0].Value = NewCurrency64(70e3) txn.FileContracts[0].MissedProofOutputs[0].Value = NewCurrency64(69e3) err = txn.correctFileContracts(30) if err != ErrFileContractOutputSumViolation { t.Error(err) } txn.FileContracts[0].MissedProofOutputs[0].Value = NewCurrency64(71e3) err = txn.correctFileContracts(30) if err != ErrFileContractOutputSumViolation { t.Error(err) } txn.FileContracts[0].MissedProofOutputs[0].Value = NewCurrency64(70e3) // Try the payouts when the value of the contract is too low to incur a // fee. txn.FileContracts = append(txn.FileContracts, FileContract{ WindowStart: 35, WindowEnd: 40, Payout: NewCurrency64(1e3), ValidProofOutputs: []SiacoinOutput{ {Value: NewCurrency64(1e3)}, }, MissedProofOutputs: []SiacoinOutput{ {Value: NewCurrency64(1e3)}, }, }) err = txn.correctFileContracts(30) if err != nil { t.Error(err) } } // TestCorrectFileContractRevisions probes the correctFileContractRevisions // method of the Transaction type. func TestCorrectFileContractRevisions(t *testing.T) { // Try a revision that starts in the past. txn := Transaction{ FileContractRevisions: []FileContractRevision{{}}, } err := txn.correctFileContractRevisions(0) if err != ErrFileContractWindowStartViolation { t.Error(err) } // Try a revision that has a window which ends before it starts. txn = Transaction{ FileContractRevisions: []FileContractRevision{ {NewWindowStart: 1}, }, } err = txn.correctFileContractRevisions(0) if err != ErrFileContractWindowEndViolation { t.Error(err) } // Try a revision with misaligned payouts. txn.FileContractRevisions = []FileContractRevision{ { NewWindowStart: 1, NewWindowEnd: 2, NewMissedProofOutputs: []SiacoinOutput{ {Value: NewCurrency64(10)}, }, }, } err = txn.correctFileContractRevisions(0) if err != ErrFileContractOutputSumViolation { t.Error("Expecting ErrFileContractOutputSumViolation:", err) } } // TestTransactionFitsInABlock probes the fitsInABlock method of the // Transaction type. func TestTransactionFitsInABlock(t *testing.T) { // Try a transaction that will fit in a block, followed by one that won't. data := make([]byte, BlockSizeLimit/2) txn := Transaction{ArbitraryData: [][]byte{data}} err := txn.fitsInABlock() if err != nil { t.Error(err) } data = make([]byte, BlockSizeLimit) txn.ArbitraryData[0] = data err = txn.fitsInABlock() if err != ErrTransactionTooLarge { t.Error(err) } } // TestTransactionFollowsMinimumValues probes the followsMinimumValues method // of the Transaction type. func TestTransactionFollowsMinimumValues(t *testing.T) { // Start with a transaction that follows all of minimum-values rules. txn := Transaction{ SiacoinOutputs: []SiacoinOutput{{Value: NewCurrency64(1)}}, FileContracts: []FileContract{{Payout: NewCurrency64(1)}}, SiafundOutputs: []SiafundOutput{{Value: NewCurrency64(1)}}, MinerFees: []Currency{NewCurrency64(1)}, } err := txn.followsMinimumValues() if err != nil { t.Error(err) } // Try a zero value for each type. txn.SiacoinOutputs[0].Value = ZeroCurrency err = txn.followsMinimumValues() if err != ErrZeroOutput { t.Error(err) } txn.SiacoinOutputs[0].Value = NewCurrency64(1) txn.FileContracts[0].Payout = ZeroCurrency err = txn.followsMinimumValues() if err != ErrZeroOutput { t.Error(err) } txn.FileContracts[0].Payout = NewCurrency64(1) txn.SiafundOutputs[0].Value = ZeroCurrency err = txn.followsMinimumValues() if err != ErrZeroOutput { t.Error(err) } txn.SiafundOutputs[0].Value = NewCurrency64(1) txn.MinerFees[0] = ZeroCurrency err = txn.followsMinimumValues() if err != ErrZeroMinerFee { t.Error(err) } txn.MinerFees[0] = NewCurrency64(1) // Try a non-zero value for the ClaimStart field of a siafund output. txn.SiafundOutputs[0].ClaimStart = NewCurrency64(1) err = txn.followsMinimumValues() if err != ErrNonZeroClaimStart { t.Error(err) } txn.SiafundOutputs[0].ClaimStart = ZeroCurrency } // TestTransactionFollowsStorageProofRules probes the followsStorageProofRules // method of the Transaction type. func TestTransactionFollowsStorageProofRules(t *testing.T) { // Try a transaction with no storage proofs. txn := Transaction{} err := txn.followsStorageProofRules() if err != nil { t.Error(err) } // Try a transaction with a legal storage proof. txn.StorageProofs = append(txn.StorageProofs, StorageProof{}) err = txn.followsStorageProofRules() if err != nil { t.Error(err) } // Try a transaction with a storage proof and a SiacoinOutput. txn.SiacoinOutputs = append(txn.SiacoinOutputs, SiacoinOutput{}) err = txn.followsStorageProofRules() if err != ErrStorageProofWithOutputs { t.Error(err) } txn.SiacoinOutputs = nil // Try a transaction with a storage proof and a FileContract. txn.FileContracts = append(txn.FileContracts, FileContract{}) err = txn.followsStorageProofRules() if err != ErrStorageProofWithOutputs { t.Error(err) } txn.FileContracts = nil // Try a transaction with a storage proof and a FileContractRevision. txn.FileContractRevisions = append(txn.FileContractRevisions, FileContractRevision{}) err = txn.followsStorageProofRules() if err != ErrStorageProofWithOutputs { t.Error(err) } txn.FileContractRevisions = nil // Try a transaction with a storage proof and a FileContractRevision. txn.SiafundOutputs = append(txn.SiafundOutputs, SiafundOutput{}) err = txn.followsStorageProofRules() if err != ErrStorageProofWithOutputs { t.Error(err) } txn.SiafundOutputs = nil } // TestTransactionNoRepeats probes the noRepeats method of the Transaction // type. func TestTransactionNoRepeats(t *testing.T) { // Try a transaction all the repeatable types but no conflicts. txn := Transaction{ SiacoinInputs: []SiacoinInput{{}}, StorageProofs: []StorageProof{{}}, FileContractRevisions: []FileContractRevision{{}}, SiafundInputs: []SiafundInput{{}}, } txn.FileContractRevisions[0].ParentID[0] = 1 // Otherwise it will conflict with the storage proof. err := txn.noRepeats() if err != nil { t.Error(err) } // Try a transaction double spending a siacoin output. txn.SiacoinInputs = append(txn.SiacoinInputs, SiacoinInput{}) err = txn.noRepeats() if err != ErrDoubleSpend { t.Error(err) } txn.SiacoinInputs = txn.SiacoinInputs[:1] // Try double spending a file contract, checking that both storage proofs // and terminations can conflict with each other. txn.StorageProofs = append(txn.StorageProofs, StorageProof{}) err = txn.noRepeats() if err != ErrDoubleSpend { t.Error(err) } txn.StorageProofs = txn.StorageProofs[:1] // Have the storage proof conflict with the file contract termination. txn.StorageProofs[0].ParentID[0] = 1 err = txn.noRepeats() if err != ErrDoubleSpend { t.Error(err) } txn.StorageProofs[0].ParentID[0] = 0 // Have the file contract termination conflict with itself. txn.FileContractRevisions = append(txn.FileContractRevisions, FileContractRevision{}) txn.FileContractRevisions[1].ParentID[0] = 1 err = txn.noRepeats() if err != ErrDoubleSpend { t.Error(err) } txn.FileContractRevisions = txn.FileContractRevisions[:1] // Try a transaction double spending a siafund output. txn.SiafundInputs = append(txn.SiafundInputs, SiafundInput{}) err = txn.noRepeats() if err != ErrDoubleSpend { t.Error(err) } txn.SiafundInputs = txn.SiafundInputs[:1] } // TestValudUnlockConditions probes the validUnlockConditions function. func TestValidUnlockConditions(t *testing.T) { // The only thing to check is the timelock. uc := UnlockConditions{Timelock: 3} err := validUnlockConditions(uc, 2) if err != ErrTimelockNotSatisfied { t.Error(err) } err = validUnlockConditions(uc, 3) if err != nil { t.Error(err) } err = validUnlockConditions(uc, 4) if err != nil { t.Error(err) } } // TestTransactionValidUnlockConditions probes the validUnlockConditions method // of the transaction type. func TestTransactionValidUnlockConditions(t *testing.T) { // Create a transaction with each type of valid unlock condition. txn := Transaction{ SiacoinInputs: []SiacoinInput{ {UnlockConditions: UnlockConditions{Timelock: 3}}, }, FileContractRevisions: []FileContractRevision{ {UnlockConditions: UnlockConditions{Timelock: 3}}, }, SiafundInputs: []SiafundInput{ {UnlockConditions: UnlockConditions{Timelock: 3}}, }, } err := txn.validUnlockConditions(4) if err != nil { t.Error(err) } // Try with illegal conditions in the siacoin inputs. txn.SiacoinInputs[0].UnlockConditions.Timelock = 5 err = txn.validUnlockConditions(4) if err == nil { t.Error(err) } txn.SiacoinInputs[0].UnlockConditions.Timelock = 3 // Try with illegal conditions in the siafund inputs. txn.FileContractRevisions[0].UnlockConditions.Timelock = 5 err = txn.validUnlockConditions(4) if err == nil { t.Error(err) } txn.FileContractRevisions[0].UnlockConditions.Timelock = 3 // Try with illegal conditions in the siafund inputs. txn.SiafundInputs[0].UnlockConditions.Timelock = 5 err = txn.validUnlockConditions(4) if err == nil { t.Error(err) } txn.SiafundInputs[0].UnlockConditions.Timelock = 3 } // TestTransactionStandaloneValid probes the StandaloneValid method of the // Transaction type. func TestTransactionStandaloneValid(t *testing.T) { // Build a working transaction. var txn Transaction err := txn.StandaloneValid(0) if err != nil { t.Error(err) } // Violate fitsInABlock. data := make([]byte, BlockSizeLimit) txn.ArbitraryData = [][]byte{data} err = txn.StandaloneValid(0) if err == nil { t.Error("failed to trigger fitsInABlock error") } txn.ArbitraryData = nil // Violate followsStorageProofRules txn.StorageProofs = []StorageProof{{}} txn.SiacoinOutputs = []SiacoinOutput{{}} txn.SiacoinOutputs[0].Value = NewCurrency64(1) err = txn.StandaloneValid(0) if err == nil { t.Error("failed to trigger followsStorageProofRules error") } txn.StorageProofs = nil txn.SiacoinOutputs = nil // Violate noRepeats txn.SiacoinInputs = []SiacoinInput{{}, {}} err = txn.StandaloneValid(0) if err == nil { t.Error("failed to trigger noRepeats error") } txn.SiacoinInputs = nil // Violate followsMinimumValues txn.SiacoinOutputs = []SiacoinOutput{{}} err = txn.StandaloneValid(0) if err == nil { t.Error("failed to trigger followsMinimumValues error") } txn.SiacoinOutputs = nil // Violate correctFileContracts txn.FileContracts = []FileContract{ { Payout: NewCurrency64(1), WindowStart: 5, WindowEnd: 5, }, } err = txn.StandaloneValid(0) if err == nil { t.Error("failed to trigger correctFileContracts error") } txn.FileContracts = nil // Violate correctFileContractRevisions txn.FileContractRevisions = []FileContractRevision{{}} err = txn.StandaloneValid(0) if err == nil { t.Error("failed to trigger correctFileContractRevisions error") } txn.FileContractRevisions = nil // Violate validUnlockConditions txn.SiacoinInputs = []SiacoinInput{{}} txn.SiacoinInputs[0].UnlockConditions.Timelock = 1 err = txn.StandaloneValid(0) if err == nil { t.Error("failed to trigger validUnlockConditions error") } txn.SiacoinInputs = nil // Violate validSignatures txn.TransactionSignatures = []TransactionSignature{{}} err = txn.StandaloneValid(0) if err == nil { t.Error("failed to trigger validSignatures error") } txn.TransactionSignatures = nil } Sia-1.0.4/vendor/000077500000000000000000000000001302033070500135165ustar00rootroot00000000000000Sia-1.0.4/vendor/golang.org/000077500000000000000000000000001302033070500155535ustar00rootroot00000000000000Sia-1.0.4/vendor/golang.org/x/000077500000000000000000000000001302033070500160225ustar00rootroot00000000000000Sia-1.0.4/vendor/golang.org/x/crypto/000077500000000000000000000000001302033070500173425ustar00rootroot00000000000000Sia-1.0.4/vendor/golang.org/x/crypto/LICENSE000066400000000000000000000027071302033070500203550ustar00rootroot00000000000000Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Sia-1.0.4/vendor/golang.org/x/crypto/PATENTS000066400000000000000000000024271302033070500204100ustar00rootroot00000000000000Additional IP Rights Grant (Patents) "This implementation" means the copyrightable works distributed by Google as part of the Go project. Google hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, transfer and otherwise run, modify and propagate the contents of this implementation of Go, where such license applies only to those patent claims, both currently owned or controlled by Google and acquired in the future, licensable by Google that are necessarily infringed by this implementation of Go. This grant does not include claims that would be infringed only as a consequence of further modification of this implementation. If you or your agent or exclusive licensee institute or order or agree to the institution of patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that this implementation of Go or any code incorporated within this implementation of Go constitutes direct or contributory patent infringement, or inducement of patent infringement, then any patent rights granted to you under this License for this implementation of Go shall terminate as of the date such litigation is filed. Sia-1.0.4/vendor/golang.org/x/crypto/twofish/000077500000000000000000000000001302033070500210255ustar00rootroot00000000000000Sia-1.0.4/vendor/golang.org/x/crypto/twofish/twofish.go000066400000000000000000000272261302033070500230500ustar00rootroot00000000000000// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package twofish implements Bruce Schneier's Twofish encryption algorithm. package twofish // Twofish is defined in http://www.schneier.com/paper-twofish-paper.pdf [TWOFISH] // This code is a port of the LibTom C implementation. // See http://libtom.org/?page=features&newsitems=5&whatfile=crypt. // LibTomCrypt is free for all purposes under the public domain. // It was heavily inspired by the go blowfish package. import "strconv" // BlockSize is the constant block size of Twofish. const BlockSize = 16 const mdsPolynomial = 0x169 // x^8 + x^6 + x^5 + x^3 + 1, see [TWOFISH] 4.2 const rsPolynomial = 0x14d // x^8 + x^6 + x^3 + x^2 + 1, see [TWOFISH] 4.3 // A Cipher is an instance of Twofish encryption using a particular key. type Cipher struct { s [4][256]uint32 k [40]uint32 } type KeySizeError int func (k KeySizeError) Error() string { return "crypto/twofish: invalid key size " + strconv.Itoa(int(k)) } // NewCipher creates and returns a Cipher. // The key argument should be the Twofish key, 16, 24 or 32 bytes. func NewCipher(key []byte) (*Cipher, error) { keylen := len(key) if keylen != 16 && keylen != 24 && keylen != 32 { return nil, KeySizeError(keylen) } // k is the number of 64 bit words in key k := keylen / 8 // Create the S[..] words var S [4 * 4]byte for i := 0; i < k; i++ { // Computes [y0 y1 y2 y3] = rs . [x0 x1 x2 x3 x4 x5 x6 x7] for j, rsRow := range rs { for k, rsVal := range rsRow { S[4*i+j] ^= gfMult(key[8*i+k], rsVal, rsPolynomial) } } } // Calculate subkeys c := new(Cipher) var tmp [4]byte for i := byte(0); i < 20; i++ { // A = h(p * 2x, Me) for j := range tmp { tmp[j] = 2 * i } A := h(tmp[:], key, 0) // B = rolc(h(p * (2x + 1), Mo), 8) for j := range tmp { tmp[j] = 2*i + 1 } B := h(tmp[:], key, 1) B = rol(B, 8) c.k[2*i] = A + B // K[2i+1] = (A + 2B) <<< 9 c.k[2*i+1] = rol(2*B+A, 9) } // Calculate sboxes switch k { case 2: for i := range c.s[0] { c.s[0][i] = mdsColumnMult(sbox[1][sbox[0][sbox[0][byte(i)]^S[0]]^S[4]], 0) c.s[1][i] = mdsColumnMult(sbox[0][sbox[0][sbox[1][byte(i)]^S[1]]^S[5]], 1) c.s[2][i] = mdsColumnMult(sbox[1][sbox[1][sbox[0][byte(i)]^S[2]]^S[6]], 2) c.s[3][i] = mdsColumnMult(sbox[0][sbox[1][sbox[1][byte(i)]^S[3]]^S[7]], 3) } case 3: for i := range c.s[0] { c.s[0][i] = mdsColumnMult(sbox[1][sbox[0][sbox[0][sbox[1][byte(i)]^S[0]]^S[4]]^S[8]], 0) c.s[1][i] = mdsColumnMult(sbox[0][sbox[0][sbox[1][sbox[1][byte(i)]^S[1]]^S[5]]^S[9]], 1) c.s[2][i] = mdsColumnMult(sbox[1][sbox[1][sbox[0][sbox[0][byte(i)]^S[2]]^S[6]]^S[10]], 2) c.s[3][i] = mdsColumnMult(sbox[0][sbox[1][sbox[1][sbox[0][byte(i)]^S[3]]^S[7]]^S[11]], 3) } default: for i := range c.s[0] { c.s[0][i] = mdsColumnMult(sbox[1][sbox[0][sbox[0][sbox[1][sbox[1][byte(i)]^S[0]]^S[4]]^S[8]]^S[12]], 0) c.s[1][i] = mdsColumnMult(sbox[0][sbox[0][sbox[1][sbox[1][sbox[0][byte(i)]^S[1]]^S[5]]^S[9]]^S[13]], 1) c.s[2][i] = mdsColumnMult(sbox[1][sbox[1][sbox[0][sbox[0][sbox[0][byte(i)]^S[2]]^S[6]]^S[10]]^S[14]], 2) c.s[3][i] = mdsColumnMult(sbox[0][sbox[1][sbox[1][sbox[0][sbox[1][byte(i)]^S[3]]^S[7]]^S[11]]^S[15]], 3) } } return c, nil } // BlockSize returns the Twofish block size, 16 bytes. func (c *Cipher) BlockSize() int { return BlockSize } // store32l stores src in dst in little-endian form. func store32l(dst []byte, src uint32) { dst[0] = byte(src) dst[1] = byte(src >> 8) dst[2] = byte(src >> 16) dst[3] = byte(src >> 24) return } // load32l reads a little-endian uint32 from src. func load32l(src []byte) uint32 { return uint32(src[0]) | uint32(src[1])<<8 | uint32(src[2])<<16 | uint32(src[3])<<24 } // rol returns x after a left circular rotation of y bits. func rol(x, y uint32) uint32 { return (x << (y & 31)) | (x >> (32 - (y & 31))) } // ror returns x after a right circular rotation of y bits. func ror(x, y uint32) uint32 { return (x >> (y & 31)) | (x << (32 - (y & 31))) } // The RS matrix. See [TWOFISH] 4.3 var rs = [4][8]byte{ {0x01, 0xA4, 0x55, 0x87, 0x5A, 0x58, 0xDB, 0x9E}, {0xA4, 0x56, 0x82, 0xF3, 0x1E, 0xC6, 0x68, 0xE5}, {0x02, 0xA1, 0xFC, 0xC1, 0x47, 0xAE, 0x3D, 0x19}, {0xA4, 0x55, 0x87, 0x5A, 0x58, 0xDB, 0x9E, 0x03}, } // sbox tables var sbox = [2][256]byte{ { 0xa9, 0x67, 0xb3, 0xe8, 0x04, 0xfd, 0xa3, 0x76, 0x9a, 0x92, 0x80, 0x78, 0xe4, 0xdd, 0xd1, 0x38, 0x0d, 0xc6, 0x35, 0x98, 0x18, 0xf7, 0xec, 0x6c, 0x43, 0x75, 0x37, 0x26, 0xfa, 0x13, 0x94, 0x48, 0xf2, 0xd0, 0x8b, 0x30, 0x84, 0x54, 0xdf, 0x23, 0x19, 0x5b, 0x3d, 0x59, 0xf3, 0xae, 0xa2, 0x82, 0x63, 0x01, 0x83, 0x2e, 0xd9, 0x51, 0x9b, 0x7c, 0xa6, 0xeb, 0xa5, 0xbe, 0x16, 0x0c, 0xe3, 0x61, 0xc0, 0x8c, 0x3a, 0xf5, 0x73, 0x2c, 0x25, 0x0b, 0xbb, 0x4e, 0x89, 0x6b, 0x53, 0x6a, 0xb4, 0xf1, 0xe1, 0xe6, 0xbd, 0x45, 0xe2, 0xf4, 0xb6, 0x66, 0xcc, 0x95, 0x03, 0x56, 0xd4, 0x1c, 0x1e, 0xd7, 0xfb, 0xc3, 0x8e, 0xb5, 0xe9, 0xcf, 0xbf, 0xba, 0xea, 0x77, 0x39, 0xaf, 0x33, 0xc9, 0x62, 0x71, 0x81, 0x79, 0x09, 0xad, 0x24, 0xcd, 0xf9, 0xd8, 0xe5, 0xc5, 0xb9, 0x4d, 0x44, 0x08, 0x86, 0xe7, 0xa1, 0x1d, 0xaa, 0xed, 0x06, 0x70, 0xb2, 0xd2, 0x41, 0x7b, 0xa0, 0x11, 0x31, 0xc2, 0x27, 0x90, 0x20, 0xf6, 0x60, 0xff, 0x96, 0x5c, 0xb1, 0xab, 0x9e, 0x9c, 0x52, 0x1b, 0x5f, 0x93, 0x0a, 0xef, 0x91, 0x85, 0x49, 0xee, 0x2d, 0x4f, 0x8f, 0x3b, 0x47, 0x87, 0x6d, 0x46, 0xd6, 0x3e, 0x69, 0x64, 0x2a, 0xce, 0xcb, 0x2f, 0xfc, 0x97, 0x05, 0x7a, 0xac, 0x7f, 0xd5, 0x1a, 0x4b, 0x0e, 0xa7, 0x5a, 0x28, 0x14, 0x3f, 0x29, 0x88, 0x3c, 0x4c, 0x02, 0xb8, 0xda, 0xb0, 0x17, 0x55, 0x1f, 0x8a, 0x7d, 0x57, 0xc7, 0x8d, 0x74, 0xb7, 0xc4, 0x9f, 0x72, 0x7e, 0x15, 0x22, 0x12, 0x58, 0x07, 0x99, 0x34, 0x6e, 0x50, 0xde, 0x68, 0x65, 0xbc, 0xdb, 0xf8, 0xc8, 0xa8, 0x2b, 0x40, 0xdc, 0xfe, 0x32, 0xa4, 0xca, 0x10, 0x21, 0xf0, 0xd3, 0x5d, 0x0f, 0x00, 0x6f, 0x9d, 0x36, 0x42, 0x4a, 0x5e, 0xc1, 0xe0, }, { 0x75, 0xf3, 0xc6, 0xf4, 0xdb, 0x7b, 0xfb, 0xc8, 0x4a, 0xd3, 0xe6, 0x6b, 0x45, 0x7d, 0xe8, 0x4b, 0xd6, 0x32, 0xd8, 0xfd, 0x37, 0x71, 0xf1, 0xe1, 0x30, 0x0f, 0xf8, 0x1b, 0x87, 0xfa, 0x06, 0x3f, 0x5e, 0xba, 0xae, 0x5b, 0x8a, 0x00, 0xbc, 0x9d, 0x6d, 0xc1, 0xb1, 0x0e, 0x80, 0x5d, 0xd2, 0xd5, 0xa0, 0x84, 0x07, 0x14, 0xb5, 0x90, 0x2c, 0xa3, 0xb2, 0x73, 0x4c, 0x54, 0x92, 0x74, 0x36, 0x51, 0x38, 0xb0, 0xbd, 0x5a, 0xfc, 0x60, 0x62, 0x96, 0x6c, 0x42, 0xf7, 0x10, 0x7c, 0x28, 0x27, 0x8c, 0x13, 0x95, 0x9c, 0xc7, 0x24, 0x46, 0x3b, 0x70, 0xca, 0xe3, 0x85, 0xcb, 0x11, 0xd0, 0x93, 0xb8, 0xa6, 0x83, 0x20, 0xff, 0x9f, 0x77, 0xc3, 0xcc, 0x03, 0x6f, 0x08, 0xbf, 0x40, 0xe7, 0x2b, 0xe2, 0x79, 0x0c, 0xaa, 0x82, 0x41, 0x3a, 0xea, 0xb9, 0xe4, 0x9a, 0xa4, 0x97, 0x7e, 0xda, 0x7a, 0x17, 0x66, 0x94, 0xa1, 0x1d, 0x3d, 0xf0, 0xde, 0xb3, 0x0b, 0x72, 0xa7, 0x1c, 0xef, 0xd1, 0x53, 0x3e, 0x8f, 0x33, 0x26, 0x5f, 0xec, 0x76, 0x2a, 0x49, 0x81, 0x88, 0xee, 0x21, 0xc4, 0x1a, 0xeb, 0xd9, 0xc5, 0x39, 0x99, 0xcd, 0xad, 0x31, 0x8b, 0x01, 0x18, 0x23, 0xdd, 0x1f, 0x4e, 0x2d, 0xf9, 0x48, 0x4f, 0xf2, 0x65, 0x8e, 0x78, 0x5c, 0x58, 0x19, 0x8d, 0xe5, 0x98, 0x57, 0x67, 0x7f, 0x05, 0x64, 0xaf, 0x63, 0xb6, 0xfe, 0xf5, 0xb7, 0x3c, 0xa5, 0xce, 0xe9, 0x68, 0x44, 0xe0, 0x4d, 0x43, 0x69, 0x29, 0x2e, 0xac, 0x15, 0x59, 0xa8, 0x0a, 0x9e, 0x6e, 0x47, 0xdf, 0x34, 0x35, 0x6a, 0xcf, 0xdc, 0x22, 0xc9, 0xc0, 0x9b, 0x89, 0xd4, 0xed, 0xab, 0x12, 0xa2, 0x0d, 0x52, 0xbb, 0x02, 0x2f, 0xa9, 0xd7, 0x61, 0x1e, 0xb4, 0x50, 0x04, 0xf6, 0xc2, 0x16, 0x25, 0x86, 0x56, 0x55, 0x09, 0xbe, 0x91, }, } // gfMult returns a·b in GF(2^8)/p func gfMult(a, b byte, p uint32) byte { B := [2]uint32{0, uint32(b)} P := [2]uint32{0, p} var result uint32 // branchless GF multiplier for i := 0; i < 7; i++ { result ^= B[a&1] a >>= 1 B[1] = P[B[1]>>7] ^ (B[1] << 1) } result ^= B[a&1] return byte(result) } // mdsColumnMult calculates y{col} where [y0 y1 y2 y3] = MDS · [x0] func mdsColumnMult(in byte, col int) uint32 { mul01 := in mul5B := gfMult(in, 0x5B, mdsPolynomial) mulEF := gfMult(in, 0xEF, mdsPolynomial) switch col { case 0: return uint32(mul01) | uint32(mul5B)<<8 | uint32(mulEF)<<16 | uint32(mulEF)<<24 case 1: return uint32(mulEF) | uint32(mulEF)<<8 | uint32(mul5B)<<16 | uint32(mul01)<<24 case 2: return uint32(mul5B) | uint32(mulEF)<<8 | uint32(mul01)<<16 | uint32(mulEF)<<24 case 3: return uint32(mul5B) | uint32(mul01)<<8 | uint32(mulEF)<<16 | uint32(mul5B)<<24 } panic("unreachable") } // h implements the S-box generation function. See [TWOFISH] 4.3.5 func h(in, key []byte, offset int) uint32 { var y [4]byte for x := range y { y[x] = in[x] } switch len(key) / 8 { case 4: y[0] = sbox[1][y[0]] ^ key[4*(6+offset)+0] y[1] = sbox[0][y[1]] ^ key[4*(6+offset)+1] y[2] = sbox[0][y[2]] ^ key[4*(6+offset)+2] y[3] = sbox[1][y[3]] ^ key[4*(6+offset)+3] fallthrough case 3: y[0] = sbox[1][y[0]] ^ key[4*(4+offset)+0] y[1] = sbox[1][y[1]] ^ key[4*(4+offset)+1] y[2] = sbox[0][y[2]] ^ key[4*(4+offset)+2] y[3] = sbox[0][y[3]] ^ key[4*(4+offset)+3] fallthrough case 2: y[0] = sbox[1][sbox[0][sbox[0][y[0]]^key[4*(2+offset)+0]]^key[4*(0+offset)+0]] y[1] = sbox[0][sbox[0][sbox[1][y[1]]^key[4*(2+offset)+1]]^key[4*(0+offset)+1]] y[2] = sbox[1][sbox[1][sbox[0][y[2]]^key[4*(2+offset)+2]]^key[4*(0+offset)+2]] y[3] = sbox[0][sbox[1][sbox[1][y[3]]^key[4*(2+offset)+3]]^key[4*(0+offset)+3]] } // [y0 y1 y2 y3] = MDS . [x0 x1 x2 x3] var mdsMult uint32 for i := range y { mdsMult ^= mdsColumnMult(y[i], i) } return mdsMult } // Encrypt encrypts a 16-byte block from src to dst, which may overlap. // Note that for amounts of data larger than a block, // it is not safe to just call Encrypt on successive blocks; // instead, use an encryption mode like CBC (see crypto/cipher/cbc.go). func (c *Cipher) Encrypt(dst, src []byte) { S1 := c.s[0] S2 := c.s[1] S3 := c.s[2] S4 := c.s[3] // Load input ia := load32l(src[0:4]) ib := load32l(src[4:8]) ic := load32l(src[8:12]) id := load32l(src[12:16]) // Pre-whitening ia ^= c.k[0] ib ^= c.k[1] ic ^= c.k[2] id ^= c.k[3] for i := 0; i < 8; i++ { k := c.k[8+i*4 : 12+i*4] t2 := S2[byte(ib)] ^ S3[byte(ib>>8)] ^ S4[byte(ib>>16)] ^ S1[byte(ib>>24)] t1 := S1[byte(ia)] ^ S2[byte(ia>>8)] ^ S3[byte(ia>>16)] ^ S4[byte(ia>>24)] + t2 ic = ror(ic^(t1+k[0]), 1) id = rol(id, 1) ^ (t2 + t1 + k[1]) t2 = S2[byte(id)] ^ S3[byte(id>>8)] ^ S4[byte(id>>16)] ^ S1[byte(id>>24)] t1 = S1[byte(ic)] ^ S2[byte(ic>>8)] ^ S3[byte(ic>>16)] ^ S4[byte(ic>>24)] + t2 ia = ror(ia^(t1+k[2]), 1) ib = rol(ib, 1) ^ (t2 + t1 + k[3]) } // Output with "undo last swap" ta := ic ^ c.k[4] tb := id ^ c.k[5] tc := ia ^ c.k[6] td := ib ^ c.k[7] store32l(dst[0:4], ta) store32l(dst[4:8], tb) store32l(dst[8:12], tc) store32l(dst[12:16], td) } // Decrypt decrypts a 16-byte block from src to dst, which may overlap. func (c *Cipher) Decrypt(dst, src []byte) { S1 := c.s[0] S2 := c.s[1] S3 := c.s[2] S4 := c.s[3] // Load input ta := load32l(src[0:4]) tb := load32l(src[4:8]) tc := load32l(src[8:12]) td := load32l(src[12:16]) // Undo undo final swap ia := tc ^ c.k[6] ib := td ^ c.k[7] ic := ta ^ c.k[4] id := tb ^ c.k[5] for i := 8; i > 0; i-- { k := c.k[4+i*4 : 8+i*4] t2 := S2[byte(id)] ^ S3[byte(id>>8)] ^ S4[byte(id>>16)] ^ S1[byte(id>>24)] t1 := S1[byte(ic)] ^ S2[byte(ic>>8)] ^ S3[byte(ic>>16)] ^ S4[byte(ic>>24)] + t2 ia = rol(ia, 1) ^ (t1 + k[2]) ib = ror(ib^(t2+t1+k[3]), 1) t2 = S2[byte(ib)] ^ S3[byte(ib>>8)] ^ S4[byte(ib>>16)] ^ S1[byte(ib>>24)] t1 = S1[byte(ia)] ^ S2[byte(ia>>8)] ^ S3[byte(ia>>16)] ^ S4[byte(ia>>24)] + t2 ic = rol(ic, 1) ^ (t1 + k[0]) id = ror(id^(t2+t1+k[1]), 1) } // Undo pre-whitening ia ^= c.k[0] ib ^= c.k[1] ic ^= c.k[2] id ^= c.k[3] store32l(dst[0:4], ia) store32l(dst[4:8], ib) store32l(dst[8:12], ic) store32l(dst[12:16], id) }