From 1a026f16d54e98f57f9df375d279376c156e6990 Mon Sep 17 00:00:00 2001 From: Remi Reuvekamp Date: Sun, 15 Oct 2017 10:29:47 +0000 Subject: [PATCH 01/29] Implement /logout/all (#307) Signed-off-by: Remi Reuvekamp --- .../auth/storage/devices/devices_table.go | 24 +++++++++++++++---- .../clientapi/auth/storage/devices/storage.go | 14 +++++++++++ .../dendrite/clientapi/routing/logout.go | 19 +++++++++++++++ .../dendrite/clientapi/routing/routing.go | 6 +++++ 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go index 65112cd84..62932b65f 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go @@ -57,13 +57,18 @@ const selectDeviceByTokenSQL = "" + const deleteDeviceSQL = "" + "DELETE FROM device_devices WHERE device_id = $1 AND localpart = $2" +const deleteDevicesByLocalpartSQL = "" + + "DELETE FROM device_devices WHERE localpart = $1" + // TODO: List devices? type devicesStatements struct { - insertDeviceStmt *sql.Stmt - selectDeviceByTokenStmt *sql.Stmt - deleteDeviceStmt *sql.Stmt - serverName gomatrixserverlib.ServerName + insertDeviceStmt *sql.Stmt + selectDeviceByTokenStmt *sql.Stmt + deleteDeviceStmt *sql.Stmt + deleteDevicesByLocalpartStmt *sql.Stmt + + serverName gomatrixserverlib.ServerName } func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { @@ -80,6 +85,9 @@ func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerN if s.deleteDeviceStmt, err = db.Prepare(deleteDeviceSQL); err != nil { return } + if s.deleteDevicesByLocalpartStmt, err = db.Prepare(deleteDevicesByLocalpartSQL); err != nil { + return + } s.serverName = server return } @@ -110,6 +118,14 @@ func (s *devicesStatements) deleteDevice( return err } +func (s *devicesStatements) deleteDevicesByLocalpart( + ctx context.Context, txn *sql.Tx, localpart string, +) error { + stmt := common.TxStmt(txn, s.deleteDevicesByLocalpartStmt) + _, err := stmt.ExecContext(ctx, localpart) + return err +} + func (s *devicesStatements) selectDeviceByToken( ctx context.Context, accessToken string, ) (*authtypes.Device, error) { diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go index ea7d87383..c100e8f58 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go @@ -109,3 +109,17 @@ func (d *Database) RemoveDevice( return nil }) } + +// RemoveAllDevices revokes devices by deleting the entry in the +// database matching the given user ID localpart. +// If something went wrong during the deletion, it will return the SQL error. +func (d *Database) RemoveAllDevices( + ctx context.Context, localpart string, +) error { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + if err := d.devices.deleteDevicesByLocalpart(ctx, txn, localpart); err != sql.ErrNoRows { + return err + } + return nil + }) +} diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/logout.go b/src/github.com/matrix-org/dendrite/clientapi/routing/logout.go index ff214fe57..d03e7957f 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/logout.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/logout.go @@ -50,3 +50,22 @@ func Logout( JSON: struct{}{}, } } + +// LogoutAll handles POST /logout/all +func LogoutAll( + req *http.Request, deviceDB *devices.Database, device *authtypes.Device, +) util.JSONResponse { + localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) + if err != nil { + return httputil.LogThenError(req, err) + } + + if err := deviceDB.RemoveAllDevices(req.Context(), localpart); err != nil { + return httputil.LogThenError(req, err) + } + + return util.JSONResponse{ + Code: 200, + JSON: struct{}{}, + } +} diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go b/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go index 0b9e4172a..04c183f32 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go @@ -160,6 +160,12 @@ func Setup( }), ).Methods("POST", "OPTIONS") + r0mux.Handle("/logout/all", + common.MakeAuthAPI("logout", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + return LogoutAll(req, deviceDB, device) + }), + ).Methods("POST", "OPTIONS") + // Stub endpoints required by Riot r0mux.Handle("/login", From 35b628f5bfbf886178e764fbc4788ecf937601b4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Oct 2017 13:20:24 +0100 Subject: [PATCH 02/29] Handle duplicate kafka messages (#301) The way we store the partition offsets for kafka streams means that when we start after a crash we may get the last message we processed again. This means that we have to be careful to ensure that the processing handles consecutive duplicates correctly. --- .../federationsender/consumers/roomserver.go | 16 ++++++++++------ .../federationsender/storage/storage.go | 19 +++++++++++++++++-- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/federationsender/consumers/roomserver.go b/src/github.com/matrix-org/dendrite/federationsender/consumers/roomserver.go index d172323ca..a396aaf6d 100644 --- a/src/github.com/matrix-org/dendrite/federationsender/consumers/roomserver.go +++ b/src/github.com/matrix-org/dendrite/federationsender/consumers/roomserver.go @@ -134,6 +134,14 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err return err } + if oldJoinedHosts == nil { + // This means that there is nothing to update as this is a duplicate + // message. + // This can happen if dendrite crashed between reading the message and + // persisting the stream position. + return nil + } + if ore.SendAsServer == api.DoNotSendToOtherServers { // Ignore event that we don't need to send anywhere. return nil @@ -146,13 +154,9 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err } // Send the event. - if err = s.queues.SendEvent( + return s.queues.SendEvent( &ore.Event, gomatrixserverlib.ServerName(ore.SendAsServer), joinedHostsAtEvent, - ); err != nil { - return err - } - - return nil + ) } // joinedHostsAtEvent works out a list of matrix servers that were joined to diff --git a/src/github.com/matrix-org/dendrite/federationsender/storage/storage.go b/src/github.com/matrix-org/dendrite/federationsender/storage/storage.go index fc7f830e4..ab97dc449 100644 --- a/src/github.com/matrix-org/dendrite/federationsender/storage/storage.go +++ b/src/github.com/matrix-org/dendrite/federationsender/storage/storage.go @@ -62,7 +62,10 @@ func (d *Database) prepare() error { } // UpdateRoom updates the joined hosts for a room and returns what the joined -// hosts were before the update. +// hosts were before the update, or nil if this was a duplicate message. +// This is called when we receive a message from kafka, so we pass in +// oldEventID and newEventID to check that we haven't missed any messages or +// this isn't a duplicate message. func (d *Database) UpdateRoom( ctx context.Context, roomID, oldEventID, newEventID string, @@ -70,22 +73,34 @@ func (d *Database) UpdateRoom( removeHosts []string, ) (joinedHosts []types.JoinedHost, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { - if err = d.insertRoom(ctx, txn, roomID); err != nil { + err = d.insertRoom(ctx, txn, roomID) + if err != nil { return err } + lastSentEventID, err := d.selectRoomForUpdate(ctx, txn, roomID) if err != nil { return err } + + if lastSentEventID == newEventID { + // We've handled this message before, so let's just ignore it. + // We can only get a duplicate for the last message we processed, + // so its enough just to compare the newEventID with lastSentEventID + return nil + } + if lastSentEventID != oldEventID { return types.EventIDMismatchError{ DatabaseID: lastSentEventID, RoomServerID: oldEventID, } } + joinedHosts, err = d.selectJoinedHosts(ctx, txn, roomID) if err != nil { return err } + for _, add := range addHosts { err = d.insertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName) if err != nil { From bd07447abec20fe87e2642b56a8d4248751d7dd7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Oct 2017 13:34:08 +0100 Subject: [PATCH 03/29] Fix some edge cases with /sync (#302) Including: - Handle timeout=0 correctly - Always return immediately on initial sync - Handle spurious wake ups from the notifier --- .../dendrite/syncapi/sync/notifier.go | 13 +- .../dendrite/syncapi/sync/notifier_test.go | 2 +- .../dendrite/syncapi/sync/requestpool.go | 112 ++++++++++++------ .../dendrite/syncapi/types/types.go | 10 ++ 4 files changed, 95 insertions(+), 42 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/notifier.go b/src/github.com/matrix-org/dendrite/syncapi/sync/notifier.go index 1b5419912..e0e5891de 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/notifier.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/notifier.go @@ -100,8 +100,10 @@ func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos ty } } -// WaitForEvents blocks until there are new events for this request. -func (n *Notifier) WaitForEvents(req syncRequest) types.StreamPosition { +// WaitForEvents blocks until there are events for this request after sincePos. +// In particular, it will return immediately if there are already events after +// sincePos for the request, but otherwise blocks waiting for new events. +func (n *Notifier) WaitForEvents(req syncRequest, sincePos types.StreamPosition) types.StreamPosition { // Do what synapse does: https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/notifier.py#L298 // - Bucket request into a lookup map keyed off a list of joined room IDs and separately a user ID // - Incoming events wake requests for a matching room ID @@ -117,7 +119,7 @@ func (n *Notifier) WaitForEvents(req syncRequest) types.StreamPosition { // TODO: We increment the stream position for any event, so it's possible that we return immediately // with a pos which contains no new events for this user. We should probably re-wait for events // automatically in this case. - if req.since != currentPos { + if sincePos != currentPos { n.streamLock.Unlock() return currentPos } @@ -141,6 +143,11 @@ func (n *Notifier) Load(ctx context.Context, db *storage.SyncServerDatabase) err return nil } +// CurrentPosition returns the current stream position +func (n *Notifier) CurrentPosition() types.StreamPosition { + return n.currPos +} + // setUsersJoinedToRooms marks the given users as 'joined' to the given rooms, such that new events from // these rooms will wake the given users /sync requests. This should be called prior to ANY calls to // OnNewEvent (eg on startup) to prevent racing. diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/notifier_test.go b/src/github.com/matrix-org/dendrite/syncapi/sync/notifier_test.go index 358243bc5..7aab417bc 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/notifier_test.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/notifier_test.go @@ -258,7 +258,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { func waitForEvents(n *Notifier, req syncRequest) (types.StreamPosition, error) { done := make(chan types.StreamPosition, 1) go func() { - newPos := n.WaitForEvents(req) + newPos := n.WaitForEvents(req, req.since) done <- newPos close(done) }() diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go b/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go index bd5909efc..81469087d 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go @@ -62,48 +62,76 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype "timeout": syncReq.timeout, }).Info("Incoming /sync request") - // Fork off 2 goroutines: one to do the work, and one to serve as a timeout. - // Whichever returns first is the one we will serve back to the client. - timeoutChan := make(chan struct{}) - timer := time.AfterFunc(syncReq.timeout, func() { - close(timeoutChan) // signal that the timeout has expired - }) + currPos := rp.notifier.CurrentPosition() - done := make(chan util.JSONResponse) - go func() { - currentPos := rp.notifier.WaitForEvents(*syncReq) - // We stop the timer BEFORE calculating the response so the cpu work - // done to calculate the response is not timed. This stops us from - // doing lots of work then timing out and sending back an empty response. - timer.Stop() - syncData, err := rp.currentSyncForUser(*syncReq, currentPos) - var res util.JSONResponse + // If this is an initial sync or timeout=0 we return immediately + if syncReq.since == types.StreamPosition(0) || syncReq.timeout == 0 { + syncData, err := rp.currentSyncForUser(*syncReq, currPos) if err != nil { - res = httputil.LogThenError(req, err) - } else { - syncData, err = rp.appendAccountData(syncData, device.UserID, *syncReq, currentPos) - if err != nil { - res = httputil.LogThenError(req, err) - } else { - res = util.JSONResponse{ - Code: 200, - JSON: syncData, - } - } + return httputil.LogThenError(req, err) } - done <- res - close(done) - }() - - select { - case <-timeoutChan: // timeout fired return util.JSONResponse{ Code: 200, - JSON: types.NewResponse(syncReq.since), + JSON: syncData, } - case res := <-done: // received a response - return res } + + // Otherwise, we wait for the notifier to tell us if something *may* have + // happened. We loop in case it turns out that nothing did happen. + + timer := time.NewTimer(syncReq.timeout) // case of timeout=0 is handled above + defer timer.Stop() + + for { + select { + // Wait for notifier to wake us up + case currPos = <-rp.makeNotifyChannel(*syncReq, currPos): + // Or for timeout to expire + case <-timer.C: + return util.JSONResponse{ + Code: 200, + JSON: types.NewResponse(syncReq.since), + } + // Or for the request to be cancelled + case <-req.Context().Done(): + return httputil.LogThenError(req, req.Context().Err()) + } + + // Note that we don't time out during calculation of sync + // response. This ensures that we don't waste the hard work + // of calculating the sync only to get timed out before we + // can respond + + syncData, err := rp.currentSyncForUser(*syncReq, currPos) + if err != nil { + return httputil.LogThenError(req, err) + } + if !syncData.IsEmpty() { + return util.JSONResponse{ + Code: 200, + JSON: syncData, + } + } + + } +} + +// makeNotifyChannel returns a channel that produces the current stream position +// when there *may* be something to return to the client. Only produces a single +// value and then closes the channel. +func (rp *RequestPool) makeNotifyChannel(syncReq syncRequest, sincePos types.StreamPosition) chan types.StreamPosition { + notified := make(chan types.StreamPosition) + + // TODO(#303): We need to ensure that WaitForEvents gets properly cancelled + // when the request is finished, or use some other mechanism to ensure we + // don't leak goroutines here + go (func() { + currentPos := rp.notifier.WaitForEvents(syncReq, sincePos) + notified <- currentPos + close(notified) + })() + + return notified } type stateEventInStateResp struct { @@ -196,12 +224,20 @@ func (rp *RequestPool) OnIncomingStateTypeRequest(req *http.Request, roomID stri } } -func (rp *RequestPool) currentSyncForUser(req syncRequest, currentPos types.StreamPosition) (*types.Response, error) { +func (rp *RequestPool) currentSyncForUser(req syncRequest, currentPos types.StreamPosition) (res *types.Response, err error) { // TODO: handle ignored users if req.since == types.StreamPosition(0) { - return rp.db.CompleteSync(req.ctx, req.userID, req.limit) + res, err = rp.db.CompleteSync(req.ctx, req.userID, req.limit) + } else { + res, err = rp.db.IncrementalSync(req.ctx, req.userID, req.since, currentPos, req.limit) } - return rp.db.IncrementalSync(req.ctx, req.userID, req.since, currentPos, req.limit) + + if err != nil { + return + } + + res, err = rp.appendAccountData(res, req.userID, req, currentPos) + return } func (rp *RequestPool) appendAccountData( diff --git a/src/github.com/matrix-org/dendrite/syncapi/types/types.go b/src/github.com/matrix-org/dendrite/syncapi/types/types.go index f710c6d5d..d0b1c38ab 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/types/types.go +++ b/src/github.com/matrix-org/dendrite/syncapi/types/types.go @@ -74,6 +74,16 @@ func NewResponse(pos StreamPosition) *Response { return &res } +// IsEmpty returns true if the response is empty, i.e. used to decided whether +// to return the response immediately to the client or to wait for more data. +func (r *Response) IsEmpty() bool { + return len(r.Rooms.Join) == 0 && + len(r.Rooms.Invite) == 0 && + len(r.Rooms.Leave) == 0 && + len(r.AccountData.Events) == 0 && + len(r.Presence.Events) == 0 +} + // JoinResponse represents a /sync response for a room which is under the 'join' key. type JoinResponse struct { State struct { From 7b0f62208ffefabefd3d12583348b5f0422c7e4a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Oct 2017 17:11:00 +0100 Subject: [PATCH 04/29] Mention how to run the linters in documentation (#310) Closes #309 --- CODE_STYLE.md | 4 ++++ CONTRIBUTING.md | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CODE_STYLE.md b/CODE_STYLE.md index 0e0a043ad..65d0daf4c 100644 --- a/CODE_STYLE.md +++ b/CODE_STYLE.md @@ -15,6 +15,10 @@ for that line or statement using a [comment directive](https://github.com/alecth `// nolint: gocyclo`. This should be used sparingly and only when its clear that the lint warning is spurious. +The linters are vendored, and can be run using [scripts/find-lint.sh](scripts/find-lint.sh) +(see file for docs) or as part of a build/test/lint cycle using +[scripts/build-test-lint.sh](scripts/build-test-lint.sh). + ## HTTP Error Handling diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 616588a59..256106faa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,8 @@ instance of dendrite, and [CODE_STYLE.md](CODE_STYLE.md) for the code style guide. We use `gb` for managing our dependencies, so `gb build` and `gb test` is how -to build dendrite and run the unit tests respectively. +to build dendrite and run the unit tests respectively. There are [scripts](scripts) +for [linting](scripts/find-lint.sh) and doing a [build/test/lint run](scripts/build-test-lint.sh). ## Picking Things To Do From 5a6a950ee842c4382ad31991281342bdc82d5953 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Oct 2017 17:13:41 +0100 Subject: [PATCH 05/29] Factor out addRoomDeltaToResponse from IncrementalSync (#306) --- .../dendrite/syncapi/storage/syncserver.go | 85 +++++++++++-------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go b/src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go index 8a0597398..8e4c24e3c 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go +++ b/src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go @@ -229,44 +229,10 @@ func (d *SyncServerDatabase) IncrementalSync( res := types.NewResponse(toPos) for _, delta := range deltas { - endPos := toPos - if delta.membershipPos > 0 && delta.membership == "leave" { - // make sure we don't leak recent events after the leave event. - // TODO: History visibility makes this somewhat complex to handle correctly. For example: - // TODO: This doesn't work for join -> leave in a single /sync request (see events prior to join). - // TODO: This will fail on join -> leave -> sensitive msg -> join -> leave - // in a single /sync request - // This is all "okay" assuming history_visibility == "shared" which it is by default. - endPos = delta.membershipPos - } - var recentStreamEvents []streamEvent - recentStreamEvents, err = d.events.selectRecentEvents( - ctx, txn, delta.roomID, fromPos, endPos, numRecentEventsPerRoom, - ) + err = d.addRoomDeltaToResponse(ctx, txn, fromPos, toPos, delta, numRecentEventsPerRoom, res) if err != nil { return nil, err } - recentEvents := streamEventsToEvents(recentStreamEvents) - delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back - - switch delta.membership { - case "join": - jr := types.NewJoinResponse() - jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) - jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Join[delta.roomID] = *jr - case "leave": - fallthrough // transitions to leave are the same as ban - case "ban": - // TODO: recentEvents may contain events that this user is not allowed to see because they are - // no longer in the room. - lr := types.NewLeaveResponse() - lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) - lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - lr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Leave[delta.roomID] = *lr - } } // TODO: This should be done in getStateDeltas @@ -418,6 +384,55 @@ func (d *SyncServerDatabase) addInvitesToResponse( return nil } +// addRoomDeltaToResponse adds a room state delta to a sync response +func (d *SyncServerDatabase) addRoomDeltaToResponse( + ctx context.Context, txn *sql.Tx, + fromPos, toPos types.StreamPosition, + delta stateDelta, + numRecentEventsPerRoom int, + res *types.Response, +) error { + endPos := toPos + if delta.membershipPos > 0 && delta.membership == "leave" { + // make sure we don't leak recent events after the leave event. + // TODO: History visibility makes this somewhat complex to handle correctly. For example: + // TODO: This doesn't work for join -> leave in a single /sync request (see events prior to join). + // TODO: This will fail on join -> leave -> sensitive msg -> join -> leave + // in a single /sync request + // This is all "okay" assuming history_visibility == "shared" which it is by default. + endPos = delta.membershipPos + } + recentStreamEvents, err := d.events.selectRecentEvents( + ctx, txn, delta.roomID, fromPos, endPos, numRecentEventsPerRoom, + ) + if err != nil { + return err + } + recentEvents := streamEventsToEvents(recentStreamEvents) + delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back + + switch delta.membership { + case "join": + jr := types.NewJoinResponse() + jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true + jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + res.Rooms.Join[delta.roomID] = *jr + case "leave": + fallthrough // transitions to leave are the same as ban + case "ban": + // TODO: recentEvents may contain events that this user is not allowed to see because they are + // no longer in the room. + lr := types.NewLeaveResponse() + lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true + lr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + res.Rooms.Leave[delta.roomID] = *lr + } + + return nil +} + // fetchStateEvents converts the set of event IDs into a set of events. It will fetch any which are missing from the database. // Returns a map of room ID to list of events. func (d *SyncServerDatabase) fetchStateEvents( From 6d15aec8d31d8dc565fb8fc2137bd20edc8fab3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20T=C3=B6tterman?= Date: Tue, 17 Oct 2017 21:12:54 +0300 Subject: [PATCH 06/29] Add /devices/ and /device/{deviceID} (#313) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paul Tötterman --- .../auth/storage/devices/devices_table.go | 55 ++++++++++- .../clientapi/auth/storage/devices/storage.go | 15 +++ .../dendrite/clientapi/routing/device.go | 97 +++++++++++++++++++ .../dendrite/clientapi/routing/routing.go | 13 +++ 4 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 src/github.com/matrix-org/dendrite/clientapi/routing/device.go diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go index 62932b65f..a614eb541 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go @@ -54,6 +54,12 @@ const insertDeviceSQL = "" + const selectDeviceByTokenSQL = "" + "SELECT device_id, localpart FROM device_devices WHERE access_token = $1" +const selectDeviceByIDSQL = "" + + "SELECT created_ts FROM device_devices WHERE localpart = $1 and device_id = $2" + +const selectDevicesByLocalpartSQL = "" + + "SELECT device_id FROM device_devices WHERE localpart = $1" + const deleteDeviceSQL = "" + "DELETE FROM device_devices WHERE device_id = $1 AND localpart = $2" @@ -65,10 +71,11 @@ const deleteDevicesByLocalpartSQL = "" + type devicesStatements struct { insertDeviceStmt *sql.Stmt selectDeviceByTokenStmt *sql.Stmt + selectDeviceByIDStmt *sql.Stmt + selectDevicesByLocalpartStmt *sql.Stmt deleteDeviceStmt *sql.Stmt deleteDevicesByLocalpartStmt *sql.Stmt - - serverName gomatrixserverlib.ServerName + serverName gomatrixserverlib.ServerName } func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { @@ -82,6 +89,12 @@ func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerN if s.selectDeviceByTokenStmt, err = db.Prepare(selectDeviceByTokenSQL); err != nil { return } + if s.selectDeviceByIDStmt, err = db.Prepare(selectDeviceByIDSQL); err != nil { + return + } + if s.selectDevicesByLocalpartStmt, err = db.Prepare(selectDevicesByLocalpartSQL); err != nil { + return + } if s.deleteDeviceStmt, err = db.Prepare(deleteDeviceSQL); err != nil { return } @@ -140,6 +153,44 @@ func (s *devicesStatements) selectDeviceByToken( return &dev, err } +func (s *devicesStatements) selectDeviceByID( + ctx context.Context, localpart, deviceID string, +) (*authtypes.Device, error) { + var dev authtypes.Device + var created int64 + stmt := s.selectDeviceByIDStmt + err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&created) + if err == nil { + dev.ID = deviceID + dev.UserID = makeUserID(localpart, s.serverName) + } + return &dev, err +} + +func (s *devicesStatements) selectDevicesByLocalpart( + ctx context.Context, localpart string, +) ([]authtypes.Device, error) { + devices := []authtypes.Device{} + + rows, err := s.selectDevicesByLocalpartStmt.QueryContext(ctx, localpart) + + if err != nil { + return devices, err + } + + for rows.Next() { + var dev authtypes.Device + err = rows.Scan(&dev.ID) + if err != nil { + return devices, err + } + dev.UserID = makeUserID(localpart, s.serverName) + devices = append(devices, dev) + } + + return devices, nil +} + func makeUserID(localpart string, server gomatrixserverlib.ServerName) string { return fmt.Sprintf("@%s:%s", localpart, string(server)) } diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go index c100e8f58..dd98bb609 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go @@ -52,6 +52,21 @@ func (d *Database) GetDeviceByAccessToken( return d.devices.selectDeviceByToken(ctx, token) } +// GetDeviceByID returns the device matching the given ID. +// Returns sql.ErrNoRows if no matching device was found. +func (d *Database) GetDeviceByID( + ctx context.Context, localpart, deviceID string, +) (*authtypes.Device, error) { + return d.devices.selectDeviceByID(ctx, localpart, deviceID) +} + +// GetDevicesByLocalpart returns the devices matching the given localpart. +func (d *Database) GetDevicesByLocalpart( + ctx context.Context, localpart string, +) ([]authtypes.Device, error) { + return d.devices.selectDevicesByLocalpart(ctx, localpart) +} + // CreateDevice makes a new device associated with the given user ID localpart. // If there is already a device with the same device ID for this user, that access token will be revoked // and replaced with the given accessToken. If the given accessToken is already in use for another device, diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/device.go b/src/github.com/matrix-org/dendrite/clientapi/routing/device.go new file mode 100644 index 000000000..9cb63bac9 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/device.go @@ -0,0 +1,97 @@ +// Copyright 2017 Paul Tötterman +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "database/sql" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +type deviceJSON struct { + DeviceID string `json:"device_id"` + UserID string `json:"user_id"` +} + +type devicesJSON struct { + Devices []deviceJSON `json:"devices"` +} + +// GetDeviceByID handles /device/{deviceID} +func GetDeviceByID( + req *http.Request, deviceDB *devices.Database, device *authtypes.Device, + deviceID string, +) util.JSONResponse { + localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) + if err != nil { + return httputil.LogThenError(req, err) + } + + ctx := req.Context() + dev, err := deviceDB.GetDeviceByID(ctx, localpart, deviceID) + if err == sql.ErrNoRows { + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound("Unknown device"), + } + } else if err != nil { + return httputil.LogThenError(req, err) + } + + return util.JSONResponse{ + Code: 200, + JSON: deviceJSON{ + DeviceID: dev.ID, + UserID: dev.UserID, + }, + } +} + +// GetDevicesByLocalpart handles /devices +func GetDevicesByLocalpart( + req *http.Request, deviceDB *devices.Database, device *authtypes.Device, +) util.JSONResponse { + localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) + if err != nil { + return httputil.LogThenError(req, err) + } + + ctx := req.Context() + devices, err := deviceDB.GetDevicesByLocalpart(ctx, localpart) + + if err != nil { + return httputil.LogThenError(req, err) + } + + res := devicesJSON{} + + for _, dev := range devices { + res.Devices = append(res.Devices, deviceJSON{ + DeviceID: dev.ID, + UserID: dev.UserID, + }) + } + + return util.JSONResponse{ + Code: 200, + JSON: res, + } +} diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go b/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go index 04c183f32..ebf48ad6e 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go @@ -355,6 +355,19 @@ func Setup( }), ).Methods("PUT", "OPTIONS") + r0mux.Handle("/devices", + common.MakeAuthAPI("get_devices", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + return GetDevicesByLocalpart(req, deviceDB, device) + }), + ).Methods("GET") + + r0mux.Handle("/device/{deviceID}", + common.MakeAuthAPI("get_device", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars := mux.Vars(req) + return GetDeviceByID(req, deviceDB, device, vars["deviceID"]) + }), + ).Methods("GET") + // Stub implementations for sytest r0mux.Handle("/events", common.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse { From ce562d96f6a3f8918cd9296fe496f31a2c25689f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Oct 2017 10:36:45 +0100 Subject: [PATCH 07/29] Don't add empty room entries to sync responses (#312) --- .../matrix-org/dendrite/syncapi/storage/syncserver.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go b/src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go index 8e4c24e3c..1a18d9374 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go +++ b/src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go @@ -411,6 +411,11 @@ func (d *SyncServerDatabase) addRoomDeltaToResponse( recentEvents := streamEventsToEvents(recentStreamEvents) delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back + // Don't bother appending empty room entries + if len(recentEvents) == 0 && len(delta.stateEvents) == 0 { + return nil + } + switch delta.membership { case "join": jr := types.NewJoinResponse() From 3790a8da40fd265827d18a2bed8b0f4e1119d03f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 25 Oct 2017 11:27:44 +0100 Subject: [PATCH 08/29] Move federationapi.readers package into routing (#314) This seems to have been missed when we folded all the readers and writers packages into routing. --- .../dendrite/federationapi/{readers => routing}/events.go | 2 +- .../dendrite/federationapi/{readers => routing}/keys.go | 2 +- .../matrix-org/dendrite/federationapi/routing/routing.go | 7 +++---- .../dendrite/federationapi/{readers => routing}/version.go | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) rename src/github.com/matrix-org/dendrite/federationapi/{readers => routing}/events.go (99%) rename src/github.com/matrix-org/dendrite/federationapi/{readers => routing}/keys.go (99%) rename src/github.com/matrix-org/dendrite/federationapi/{readers => routing}/version.go (98%) diff --git a/src/github.com/matrix-org/dendrite/federationapi/readers/events.go b/src/github.com/matrix-org/dendrite/federationapi/routing/events.go similarity index 99% rename from src/github.com/matrix-org/dendrite/federationapi/readers/events.go rename to src/github.com/matrix-org/dendrite/federationapi/routing/events.go index bd4817efe..0b2fdcf54 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/readers/events.go +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/events.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package readers +package routing import ( "context" diff --git a/src/github.com/matrix-org/dendrite/federationapi/readers/keys.go b/src/github.com/matrix-org/dendrite/federationapi/routing/keys.go similarity index 99% rename from src/github.com/matrix-org/dendrite/federationapi/readers/keys.go rename to src/github.com/matrix-org/dendrite/federationapi/routing/keys.go index b28bdec99..ea44e4c05 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/readers/keys.go +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/keys.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package readers +package routing import ( "encoding/json" diff --git a/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go b/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go index 91668af34..c22dd1879 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/federationapi/readers" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -48,7 +47,7 @@ func Setup( v1fedmux := apiMux.PathPrefix(pathPrefixV1Federation).Subrouter() localKeys := common.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse { - return readers.LocalKeys(cfg) + return LocalKeys(cfg) }) // Ignore the {keyID} argument as we only have a single server key so we always @@ -100,7 +99,7 @@ func Setup( "federation_get_event", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars := mux.Vars(httpReq) - return readers.GetEvent( + return GetEvent( httpReq.Context(), request, cfg, query, time.Now(), keys, vars["eventID"], ) }, @@ -109,7 +108,7 @@ func Setup( v1fedmux.Handle("/version", common.MakeExternalAPI( "federation_version", func(httpReq *http.Request) util.JSONResponse { - return readers.Version() + return Version() }, )).Methods("GET") } diff --git a/src/github.com/matrix-org/dendrite/federationapi/readers/version.go b/src/github.com/matrix-org/dendrite/federationapi/routing/version.go similarity index 98% rename from src/github.com/matrix-org/dendrite/federationapi/readers/version.go rename to src/github.com/matrix-org/dendrite/federationapi/routing/version.go index 5af082214..082af533e 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/readers/version.go +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/version.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package readers +package routing import ( "github.com/matrix-org/util" From e5944e0fdbdf6a908926e5df3b753b63370c03f8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 25 Oct 2017 14:44:33 +0100 Subject: [PATCH 09/29] Move BuildEvent to common package (#315) This is in preperation for implementing various federation APIs that need to build events. --- .../matrix-org/dendrite/clientapi/routing/joinroom.go | 6 +++--- .../matrix-org/dendrite/clientapi/routing/membership.go | 7 +++---- .../matrix-org/dendrite/clientapi/routing/profile.go | 3 +-- .../matrix-org/dendrite/clientapi/routing/sendevent.go | 6 +++--- .../matrix-org/dendrite/clientapi/threepid/invites.go | 3 +-- .../dendrite/{clientapi/events => common}/events.go | 2 +- 6 files changed, 12 insertions(+), 15 deletions(-) rename src/github.com/matrix-org/dendrite/{clientapi/events => common}/events.go (99%) diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/joinroom.go b/src/github.com/matrix-org/dendrite/clientapi/routing/joinroom.go index 890d6f1a2..73a751acf 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/joinroom.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/joinroom.go @@ -22,10 +22,10 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/events" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrix" @@ -215,7 +215,7 @@ func (r joinRoomReq) joinRoomUsingServers( } var queryRes api.QueryLatestEventsAndStateResponse - event, err := events.BuildEvent(r.req.Context(), &eb, r.cfg, r.queryAPI, &queryRes) + event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.queryAPI, &queryRes) if err == nil { if err = r.producer.SendEvents(r.req.Context(), []gomatrixserverlib.Event{*event}, r.cfg.Matrix.ServerName); err != nil { return httputil.LogThenError(r.req, err) @@ -227,7 +227,7 @@ func (r joinRoomReq) joinRoomUsingServers( }{roomID}, } } - if err != events.ErrRoomNoExists { + if err != common.ErrRoomNoExists { return httputil.LogThenError(r.req, err) } diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/membership.go b/src/github.com/matrix-org/dendrite/clientapi/routing/membership.go index fec27ae76..e94fbde7d 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/membership.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/membership.go @@ -21,7 +21,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/events" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" @@ -62,7 +61,7 @@ func SendMembership( Code: 400, JSON: jsonerror.NotTrusted(body.IDServer), } - } else if err == events.ErrRoomNoExists { + } else if err == common.ErrRoomNoExists { return util.JSONResponse{ Code: 404, JSON: jsonerror.NotFound(err.Error()), @@ -89,7 +88,7 @@ func SendMembership( Code: 400, JSON: jsonerror.BadJSON(err.Error()), } - } else if err == events.ErrRoomNoExists { + } else if err == common.ErrRoomNoExists { return util.JSONResponse{ Code: 404, JSON: jsonerror.NotFound(err.Error()), @@ -149,7 +148,7 @@ func buildMembershipEvent( return nil, err } - return events.BuildEvent(ctx, &builder, cfg, queryAPI, nil) + return common.BuildEvent(ctx, &builder, cfg, queryAPI, nil) } // loadProfile lookups the profile of a given user from the database and returns diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/profile.go b/src/github.com/matrix-org/dendrite/clientapi/routing/profile.go index 918292e59..da47451fa 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/profile.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/profile.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/events" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" @@ -285,7 +284,7 @@ func buildMembershipEvents( return nil, err } - event, err := events.BuildEvent(ctx, &builder, *cfg, queryAPI, nil) + event, err := common.BuildEvent(ctx, &builder, *cfg, queryAPI, nil) if err != nil { return nil, err } diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/sendevent.go b/src/github.com/matrix-org/dendrite/clientapi/routing/sendevent.go index d912f10ba..dc2f58f6a 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/sendevent.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/sendevent.go @@ -18,10 +18,10 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/events" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -67,8 +67,8 @@ func SendEvent( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := events.BuildEvent(req.Context(), &builder, cfg, queryAPI, &queryRes) - if err == events.ErrRoomNoExists { + e, err := common.BuildEvent(req.Context(), &builder, cfg, queryAPI, &queryRes) + if err == common.ErrRoomNoExists { return util.JSONResponse{ Code: 404, JSON: jsonerror.NotFound("Room does not exist"), diff --git a/src/github.com/matrix-org/dendrite/clientapi/threepid/invites.go b/src/github.com/matrix-org/dendrite/clientapi/threepid/invites.go index 51c0dd9c9..836591829 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/threepid/invites.go +++ b/src/github.com/matrix-org/dendrite/clientapi/threepid/invites.go @@ -26,7 +26,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/events" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" @@ -351,7 +350,7 @@ func emit3PIDInviteEvent( } var queryRes *api.QueryLatestEventsAndStateResponse - event, err := events.BuildEvent(ctx, builder, cfg, queryAPI, queryRes) + event, err := common.BuildEvent(ctx, builder, cfg, queryAPI, queryRes) if err != nil { return err } diff --git a/src/github.com/matrix-org/dendrite/clientapi/events/events.go b/src/github.com/matrix-org/dendrite/common/events.go similarity index 99% rename from src/github.com/matrix-org/dendrite/clientapi/events/events.go rename to src/github.com/matrix-org/dendrite/common/events.go index 23cb5bbc2..cf2df8422 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/events/events.go +++ b/src/github.com/matrix-org/dendrite/common/events.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package events +package common import ( "context" From 49c040c89f92894aa8839b27c87fb569f37e398e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 26 Oct 2017 11:34:54 +0100 Subject: [PATCH 10/29] Refactor Notifier to return channel (#311) This has two benefits: 1. Using channels makes it easier to time out while waiting 2. Allows us to clean up goroutines that were waiting if we timeout the request --- .../dendrite/syncapi/sync/notifier.go | 60 ++++--- .../dendrite/syncapi/sync/notifier_test.go | 15 +- .../dendrite/syncapi/sync/requestpool.go | 26 +-- .../dendrite/syncapi/sync/userstream.go | 155 ++++++++++++++---- 4 files changed, 170 insertions(+), 86 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/notifier.go b/src/github.com/matrix-org/dendrite/syncapi/sync/notifier.go index e0e5891de..4712a2c74 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/notifier.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/notifier.go @@ -17,11 +17,12 @@ package sync import ( "context" "sync" + "time" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" + log "github.com/sirupsen/logrus" ) // Notifier will wake up sleeping requests when there is some new data. @@ -38,6 +39,8 @@ type Notifier struct { currPos types.StreamPosition // A map of user_id => UserStream which can be used to wake a given user's /sync request. userStreams map[string]*UserStream + // The last time we cleaned out stale entries from the userStreams map + lastCleanUpTime time.Time } // NewNotifier creates a new notifier set to the given stream position. @@ -49,6 +52,7 @@ func NewNotifier(pos types.StreamPosition) *Notifier { roomIDToJoinedUsers: make(map[string]userIDSet), userStreams: make(map[string]*UserStream), streamLock: &sync.Mutex{}, + lastCleanUpTime: time.Now(), } } @@ -63,6 +67,8 @@ func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos ty defer n.streamLock.Unlock() n.currPos = pos + n.removeEmptyUserStreams() + if ev != nil { // Map this event's room_id to a list of joined users, and wake them up. userIDs := n.joinedUsers(ev.RoomID()) @@ -100,10 +106,10 @@ func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos ty } } -// WaitForEvents blocks until there are events for this request after sincePos. -// In particular, it will return immediately if there are already events after -// sincePos for the request, but otherwise blocks waiting for new events. -func (n *Notifier) WaitForEvents(req syncRequest, sincePos types.StreamPosition) types.StreamPosition { +// GetListener returns a UserStreamListener that can be used to wait for +// updates for a user. Must be closed. +// notify for anything before sincePos +func (n *Notifier) GetListener(req syncRequest) UserStreamListener { // Do what synapse does: https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/notifier.py#L298 // - Bucket request into a lookup map keyed off a list of joined room IDs and separately a user ID // - Incoming events wake requests for a matching room ID @@ -112,25 +118,12 @@ func (n *Notifier) WaitForEvents(req syncRequest, sincePos types.StreamPosition) // TODO: v1 /events 'peeking' has an 'explicit room ID' which is also tracked, // but given we don't do /events, let's pretend it doesn't exist. - // In a guard, check if the /sync request should block, and block it until we get woken up n.streamLock.Lock() - currentPos := n.currPos + defer n.streamLock.Unlock() - // TODO: We increment the stream position for any event, so it's possible that we return immediately - // with a pos which contains no new events for this user. We should probably re-wait for events - // automatically in this case. - if sincePos != currentPos { - n.streamLock.Unlock() - return currentPos - } + n.removeEmptyUserStreams() - // wait to be woken up, and then re-check the stream position - req.log.WithField("user_id", req.userID).Info("Waiting for event") - - // give up the stream lock prior to waiting on the user lock - stream := n.fetchUserStream(req.userID, true) - n.streamLock.Unlock() - return stream.Wait(currentPos) + return n.fetchUserStream(req.userID, true).GetListener(req.ctx) } // Load the membership states required to notify users correctly. @@ -178,7 +171,7 @@ func (n *Notifier) fetchUserStream(userID string, makeIfNotExists bool) *UserStr stream, ok := n.userStreams[userID] if !ok && makeIfNotExists { // TODO: Unbounded growth of streams (1 per user) - stream = NewUserStream(userID) + stream = NewUserStream(userID, n.currPos) n.userStreams[userID] = stream } return stream @@ -208,6 +201,29 @@ func (n *Notifier) joinedUsers(roomID string) (userIDs []string) { return n.roomIDToJoinedUsers[roomID].values() } +// removeEmptyUserStreams iterates through the user stream map and removes any +// that have been empty for a certain amount of time. This is a crude way of +// ensuring that the userStreams map doesn't grow forver. +// This should be called when the notifier gets called for whatever reason, +// the function itself is responsible for ensuring it doesn't iterate too +// often. +// NB: Callers should have locked the mutex before calling this function. +func (n *Notifier) removeEmptyUserStreams() { + // Only clean up now and again + now := time.Now() + if n.lastCleanUpTime.Add(time.Minute).After(now) { + return + } + n.lastCleanUpTime = now + + deleteBefore := now.Add(-5 * time.Minute) + for key, value := range n.userStreams { + if value.TimeOfLastNonEmpty().Before(deleteBefore) { + delete(n.userStreams, key) + } + } +} + // A string set, mainly existing for improving clarity of structs in this file. type userIDSet map[string]bool diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/notifier_test.go b/src/github.com/matrix-org/dendrite/syncapi/sync/notifier_test.go index 7aab417bc..6ee259681 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/notifier_test.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/notifier_test.go @@ -256,24 +256,22 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { // same as Notifier.WaitForEvents but with a timeout. func waitForEvents(n *Notifier, req syncRequest) (types.StreamPosition, error) { - done := make(chan types.StreamPosition, 1) - go func() { - newPos := n.WaitForEvents(req, req.since) - done <- newPos - close(done) - }() + listener := n.GetListener(req) + defer listener.Close() + select { case <-time.After(5 * time.Second): return types.StreamPosition(0), fmt.Errorf( "waitForEvents timed out waiting for %s (pos=%d)", req.userID, req.since, ) - case p := <-done: + case <-listener.GetNotifyChannel(req.since): + p := listener.GetStreamPosition() return p, nil } } // Wait until something is Wait()ing on the user stream. -func waitForBlocking(s *UserStream, numBlocking int) { +func waitForBlocking(s *UserStream, numBlocking uint) { for numBlocking != s.NumWaiting() { // This is horrible but I don't want to add a signalling mechanism JUST for testing. time.Sleep(1 * time.Microsecond) @@ -288,5 +286,6 @@ func newTestSyncRequest(userID string, since types.StreamPosition) syncRequest { wantFullState: false, limit: defaultTimelineLimit, log: util.GetLogger(context.TODO()), + ctx: context.TODO(), } } diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go b/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go index 81469087d..ebfb140d1 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go @@ -19,7 +19,6 @@ import ( "net/http" "time" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" @@ -28,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" ) // RequestPool manages HTTP long-poll connections for /sync @@ -82,10 +82,14 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype timer := time.NewTimer(syncReq.timeout) // case of timeout=0 is handled above defer timer.Stop() + userStreamListener := rp.notifier.GetListener(*syncReq) + defer userStreamListener.Close() + for { select { // Wait for notifier to wake us up - case currPos = <-rp.makeNotifyChannel(*syncReq, currPos): + case <-userStreamListener.GetNotifyChannel(currPos): + currPos = userStreamListener.GetStreamPosition() // Or for timeout to expire case <-timer.C: return util.JSONResponse{ @@ -116,24 +120,6 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype } } -// makeNotifyChannel returns a channel that produces the current stream position -// when there *may* be something to return to the client. Only produces a single -// value and then closes the channel. -func (rp *RequestPool) makeNotifyChannel(syncReq syncRequest, sincePos types.StreamPosition) chan types.StreamPosition { - notified := make(chan types.StreamPosition) - - // TODO(#303): We need to ensure that WaitForEvents gets properly cancelled - // when the request is finished, or use some other mechanism to ensure we - // don't leak goroutines here - go (func() { - currentPos := rp.notifier.WaitForEvents(syncReq, sincePos) - notified <- currentPos - close(notified) - })() - - return notified -} - type stateEventInStateResp struct { gomatrixserverlib.ClientEvent PrevContent json.RawMessage `json:"prev_content,omitempty"` diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/userstream.go b/src/github.com/matrix-org/dendrite/syncapi/sync/userstream.go index 349b3e272..77d09c202 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/userstream.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/userstream.go @@ -15,65 +15,148 @@ package sync import ( + "context" + "runtime" "sync" + "time" "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/util" ) // UserStream represents a communication mechanism between the /sync request goroutine -// and the underlying sync server goroutines. Goroutines can Wait() for a stream position and -// goroutines can Broadcast(streamPosition) to other goroutines. +// and the underlying sync server goroutines. +// Goroutines can get a UserStreamListener to wait for updates, and can Broadcast() +// updates. type UserStream struct { UserID string - // Because this is a Cond, we can notify all waiting goroutines so this works - // across devices for the same user. Protects pos. - cond *sync.Cond - // The position to broadcast to callers of Wait(). + // The lock that protects changes to this struct + lock sync.Mutex + // Closed when there is an update. + signalChannel chan struct{} + // The last stream position that there may have been an update for the suser pos types.StreamPosition - // The number of goroutines blocked on Wait() - used for testing and metrics - numWaiting int + // The last time when we had some listeners waiting + timeOfLastChannel time.Time + // The number of listeners waiting + numWaiting uint +} + +// UserStreamListener allows a sync request to wait for updates for a user. +type UserStreamListener struct { + userStream *UserStream + + // Whether the stream has been closed + hasClosed bool } // NewUserStream creates a new user stream -func NewUserStream(userID string) *UserStream { +func NewUserStream(userID string, currPos types.StreamPosition) *UserStream { return &UserStream{ - UserID: userID, - cond: sync.NewCond(&sync.Mutex{}), + UserID: userID, + timeOfLastChannel: time.Now(), + pos: currPos, + signalChannel: make(chan struct{}), } } -// Wait blocks until there is a new stream position for this user, which is then returned. -// waitAtPos should be the position the stream thinks it should be waiting at. -func (s *UserStream) Wait(waitAtPos types.StreamPosition) (pos types.StreamPosition) { - s.cond.L.Lock() - // Before we start blocking, we need to make sure that we didn't race with a call - // to Broadcast() between calling Wait() and actually sleeping. We check the last - // broadcast pos to see if it is newer than the pos we are meant to wait at. If it - // is newer, something has Broadcast to this stream more recently so return immediately. - if s.pos > waitAtPos { - pos = s.pos - s.cond.L.Unlock() - return +// GetListener returns UserStreamListener that a sync request can use to wait +// for new updates with. +// UserStreamListener must be closed +func (s *UserStream) GetListener(ctx context.Context) UserStreamListener { + s.lock.Lock() + defer s.lock.Unlock() + + s.numWaiting++ // We decrement when UserStreamListener is closed + + listener := UserStreamListener{ + userStream: s, } - s.numWaiting++ - s.cond.Wait() - pos = s.pos - s.numWaiting-- - s.cond.L.Unlock() - return + + // Lets be a bit paranoid here and check that Close() is being called + runtime.SetFinalizer(&listener, func(l *UserStreamListener) { + if !l.hasClosed { + util.GetLogger(ctx).Warn("Didn't call Close on UserStreamListener") + l.Close() + } + }) + + return listener } // Broadcast a new stream position for this user. func (s *UserStream) Broadcast(pos types.StreamPosition) { - s.cond.L.Lock() + s.lock.Lock() + defer s.lock.Unlock() + s.pos = pos - s.cond.L.Unlock() - s.cond.Broadcast() + + close(s.signalChannel) + + s.signalChannel = make(chan struct{}) } -// NumWaiting returns the number of goroutines waiting for Wait() to return. Used for metrics and testing. -func (s *UserStream) NumWaiting() int { - s.cond.L.Lock() - defer s.cond.L.Unlock() +// NumWaiting returns the number of goroutines waiting for waiting for updates. +// Used for metrics and testing. +func (s *UserStream) NumWaiting() uint { + s.lock.Lock() + defer s.lock.Unlock() return s.numWaiting } + +// TimeOfLastNonEmpty returns the last time that the number of waiting listeners +// was non-empty, may be time.Now() if number of waiting listeners is currently +// non-empty. +func (s *UserStream) TimeOfLastNonEmpty() time.Time { + s.lock.Lock() + defer s.lock.Unlock() + + if s.numWaiting > 0 { + return time.Now() + } + + return s.timeOfLastChannel +} + +// GetStreamPosition returns last stream position which the UserStream was +// notified about +func (s *UserStreamListener) GetStreamPosition() types.StreamPosition { + s.userStream.lock.Lock() + defer s.userStream.lock.Unlock() + + return s.userStream.pos +} + +// GetNotifyChannel returns a channel that is closed when there may be an +// update for the user. +// sincePos specifies from which point we want to be notified about. If there +// has already been an update after sincePos we'll return a closed channel +// immediately. +func (s *UserStreamListener) GetNotifyChannel(sincePos types.StreamPosition) <-chan struct{} { + s.userStream.lock.Lock() + defer s.userStream.lock.Unlock() + + if sincePos < s.userStream.pos { + // If the listener is behind, i.e. missed a potential update, then we + // want them to wake up immediately. We do this by returning a new + // closed stream, which returns immediately when selected. + closedChannel := make(chan struct{}) + close(closedChannel) + return closedChannel + } + + return s.userStream.signalChannel +} + +// Close cleans up resources used +func (s *UserStreamListener) Close() { + s.userStream.lock.Lock() + defer s.userStream.lock.Unlock() + + if !s.hasClosed { + s.userStream.numWaiting-- + s.userStream.timeOfLastChannel = time.Now() + } + + s.hasClosed = true +} From b7cfc2e057439334e3ddfa69cebff653f3114d00 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Sun, 5 Nov 2017 10:03:54 -0800 Subject: [PATCH 11/29] Federation: Implement Query Profile API (#317) * Federation: Implement Query Profile API Implements the server portion of: `GET /_matrix/federation/v1/query/profile?user_id=...&field=...` Closes #278 Signed-off-by: Andrew (anoa) * Properly export profile-related structs and fix wording Signed-off-by: Andrew (anoa) * Check provided user's domain matches our own. --- .../dendrite/clientapi/jsonerror/jsonerror.go | 12 +++ .../dendrite/clientapi/routing/profile.go | 23 ++--- .../matrix-org/dendrite/common/types.go | 16 ++++ .../dendrite/federationapi/routing/profile.go | 87 +++++++++++++++++++ .../dendrite/federationapi/routing/routing.go | 9 ++ 5 files changed, 129 insertions(+), 18 deletions(-) create mode 100644 src/github.com/matrix-org/dendrite/federationapi/routing/profile.go diff --git a/src/github.com/matrix-org/dendrite/clientapi/jsonerror/jsonerror.go b/src/github.com/matrix-org/dendrite/clientapi/jsonerror/jsonerror.go index 8f168f4fa..07fe90304 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/jsonerror/jsonerror.go +++ b/src/github.com/matrix-org/dendrite/clientapi/jsonerror/jsonerror.go @@ -67,6 +67,18 @@ func NotFound(msg string) *MatrixError { return &MatrixError{"M_NOT_FOUND", msg} } +// MissingArgument is an error when the client tries to access a resource +// without providing an argument that is required. +func MissingArgument(msg string) *MatrixError { + return &MatrixError{"M_MISSING_ARGUMENT", msg} +} + +// InvalidArgumentValue is an error when the client tries to provide an +// invalid value for a valid argument +func InvalidArgumentValue(msg string) *MatrixError { + return &MatrixError{"M_INVALID_ARGUMENT_VALUE", msg} +} + // MissingToken is an error when the client tries to access a resource which // requires authentication without supplying credentials. func MissingToken(msg string) *MatrixError { diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/profile.go b/src/github.com/matrix-org/dendrite/clientapi/routing/profile.go index da47451fa..1403a8292 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/profile.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/profile.go @@ -31,19 +31,6 @@ import ( "github.com/matrix-org/util" ) -type profileResponse struct { - AvatarURL string `json:"avatar_url"` - DisplayName string `json:"displayname"` -} - -type avatarURL struct { - AvatarURL string `json:"avatar_url"` -} - -type displayName struct { - DisplayName string `json:"displayname"` -} - // GetProfile implements GET /profile/{userID} func GetProfile( req *http.Request, accountDB *accounts.Database, userID string, @@ -63,7 +50,7 @@ func GetProfile( if err != nil { return httputil.LogThenError(req, err) } - res := profileResponse{ + res := common.ProfileResponse{ AvatarURL: profile.AvatarURL, DisplayName: profile.DisplayName, } @@ -86,7 +73,7 @@ func GetAvatarURL( if err != nil { return httputil.LogThenError(req, err) } - res := avatarURL{ + res := common.AvatarURL{ AvatarURL: profile.AvatarURL, } return util.JSONResponse{ @@ -110,7 +97,7 @@ func SetAvatarURL( changedKey := "avatar_url" - var r avatarURL + var r common.AvatarURL if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr } @@ -178,7 +165,7 @@ func GetDisplayName( if err != nil { return httputil.LogThenError(req, err) } - res := displayName{ + res := common.DisplayName{ DisplayName: profile.DisplayName, } return util.JSONResponse{ @@ -202,7 +189,7 @@ func SetDisplayName( changedKey := "displayname" - var r displayName + var r common.DisplayName if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr } diff --git a/src/github.com/matrix-org/dendrite/common/types.go b/src/github.com/matrix-org/dendrite/common/types.go index 471a2f30b..d8c5c5a7e 100644 --- a/src/github.com/matrix-org/dendrite/common/types.go +++ b/src/github.com/matrix-org/dendrite/common/types.go @@ -20,3 +20,19 @@ type AccountData struct { RoomID string `json:"room_id"` Type string `json:"type"` } + +// ProfileResponse is a struct containing all known user profile data +type ProfileResponse struct { + AvatarURL string `json:"avatar_url"` + DisplayName string `json:"displayname"` +} + +// AvatarURL is a struct containing only the URL to a user's avatar +type AvatarURL struct { + AvatarURL string `json:"avatar_url"` +} + +// DisplayName is a struct containing only a user's display name +type DisplayName struct { + DisplayName string `json:"displayname"` +} diff --git a/src/github.com/matrix-org/dendrite/federationapi/routing/profile.go b/src/github.com/matrix-org/dendrite/federationapi/routing/profile.go new file mode 100644 index 000000000..48f473669 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/profile.go @@ -0,0 +1,87 @@ +// Copyright 2017 New Vector Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// GetProfile implements GET /_matrix/federation/v1/query/profile +func GetProfile( + httpReq *http.Request, + accountDB *accounts.Database, + cfg config.Dendrite, +) util.JSONResponse { + userID, field := httpReq.FormValue("user_id"), httpReq.FormValue("field") + + // httpReq.FormValue will return an empty string if value is not found + if userID == "" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.MissingArgument("The request body did not contain required argument 'user_id'."), + } + } + + localpart, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return httputil.LogThenError(httpReq, err) + } + + if domain != cfg.Matrix.ServerName { + return httputil.LogThenError(httpReq, err) + } + + profile, err := accountDB.GetProfileByLocalpart(httpReq.Context(), localpart) + if err != nil { + return httputil.LogThenError(httpReq, err) + } + + var res interface{} + code := 200 + + if field != "" { + switch field { + case "displayname": + res = common.DisplayName{ + profile.DisplayName, + } + case "avatar_url": + res = common.AvatarURL{ + profile.AvatarURL, + } + default: + code = 400 + res = jsonerror.InvalidArgumentValue("The request body did not contain an allowed value of argument 'field'. Allowed values are either: 'avatar_url', 'displayname'.") + } + } else { + res = common.ProfileResponse{ + profile.AvatarURL, + profile.DisplayName, + } + } + + return util.JSONResponse{ + Code: code, + JSON: res, + } +} diff --git a/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go b/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go index c22dd1879..c50afd6e0 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go @@ -105,6 +105,15 @@ func Setup( }, )).Methods("GET") + v1fedmux.Handle("/query/profile", common.MakeFedAPI( + "federation_query_profile", cfg.Matrix.ServerName, keys, + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { + return GetProfile( + httpReq, accountDB, cfg, + ) + }, + )).Methods("GET") + v1fedmux.Handle("/version", common.MakeExternalAPI( "federation_version", func(httpReq *http.Request) util.JSONResponse { From 90396b56208372ceaf1f40e2266be5038df936df Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@googlemail.com> Date: Thu, 9 Nov 2017 09:58:45 +0000 Subject: [PATCH 12/29] implement voip/turnServer API endpoint (#323) Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- dendrite-config.yaml | 18 ++++ .../dendrite/clientapi/routing/routing.go | 8 +- .../dendrite/clientapi/routing/voip.go | 85 +++++++++++++++++++ .../dendrite/common/config/config.go | 30 +++++++ 4 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 src/github.com/matrix-org/dendrite/clientapi/routing/voip.go diff --git a/dendrite-config.yaml b/dendrite-config.yaml index cb768a106..9baba8f94 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -54,6 +54,24 @@ media: height: 600 method: scale +# The config for the TURN server +turn: + # Whether or not guests can request TURN credentials + turn_allow_guests: true + # How long the authorization should last + turn_user_lifetime: "1h" + # The list of TURN URIs to pass to clients + turn_uris: [] + + # Authorization via Shared Secret + # The shared secret from coturn + turn_shared_secret: "" + + # Authorization via Static Username & Password + # Hardcoded Username and Password + turn_username: "" + turn_password: "" + # The config for communicating with kafka kafka: # Where the kafka servers are running. diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go b/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go index ebf48ad6e..87f52ad09 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go @@ -284,12 +284,8 @@ func Setup( ).Methods("PUT", "OPTIONS") r0mux.Handle("/voip/turnServer", - common.MakeExternalAPI("turn_server", func(req *http.Request) util.JSONResponse { - // TODO: Return credentials for a turn server if one is configured. - return util.JSONResponse{ - Code: 200, - JSON: struct{}{}, - } + common.MakeAuthAPI("turn_server", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + return RequestTurnServer(req, device, cfg) }), ).Methods("GET") diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go b/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go new file mode 100644 index 000000000..e699a91f2 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go @@ -0,0 +1,85 @@ +// Copyright 2017 Michael Telatysnki <7t3chguy@gmail.com> +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "fmt" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/clientapi/httputil" +) + +type turnServerResponse struct { + Username string `json:"username"` + Password string `json:"password"` + URIs []string `json:"uris"` + TTL int `json:"ttl"` +} + +// RequestTurnServer implements: +// GET /voip/turnServer +func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg config.Dendrite) util.JSONResponse { + turnConfig := cfg.TURN + + // TODO Guest Support + if len(turnConfig.URIs) == 0 || turnConfig.UserLifetime == "" { + return util.JSONResponse{ + Code: 200, + JSON: struct{}{}, + } + } + + // Duration checked at startup, err not possible + duration, _ := time.ParseDuration(turnConfig.UserLifetime) + + resp := turnServerResponse{ + URIs: turnConfig.URIs, + TTL: int(duration.Seconds()), + } + + if turnConfig.SharedSecret != "" { + expiry := time.Now().Add(duration).Unix() + mac := hmac.New(sha1.New, []byte(turnConfig.SharedSecret)) + _, err := mac.Write([]byte(resp.Username)) + + if err != nil { + return httputil.LogThenError(req, err) + } + + resp.Username = fmt.Sprintf("%d:%s", expiry, device.UserID) + resp.Password = base64.StdEncoding.EncodeToString(mac.Sum(nil)) + } else if turnConfig.Username != "" && turnConfig.Password != "" { + resp.Username = turnConfig.Username + resp.Password = turnConfig.Password + } else { + return util.JSONResponse{ + Code: 200, + JSON: struct{}{}, + } + } + + return util.JSONResponse{ + Code: 200, + JSON: resp, + } +} diff --git a/src/github.com/matrix-org/dendrite/common/config/config.go b/src/github.com/matrix-org/dendrite/common/config/config.go index 8f80aa2bc..82bdc3dca 100644 --- a/src/github.com/matrix-org/dendrite/common/config/config.go +++ b/src/github.com/matrix-org/dendrite/common/config/config.go @@ -150,6 +150,26 @@ type Dendrite struct { PublicRoomsAPI DataSource `yaml:"public_rooms_api"` } `yaml:"database"` + // TURN Server Config + TURN struct { + // TODO Guest Support + // Whether or not guests can request TURN credentials + //AllowGuests bool `yaml:"turn_allow_guests"` + // How long the authorization should last + UserLifetime string `yaml:"turn_user_lifetime"` + // The list of TURN URIs to pass to clients + URIs []string `yaml:"turn_uris"` + + // Authorization via Shared Secret + // The shared secret from coturn + SharedSecret string `yaml:"turn_shared_secret"` + + // Authorization via Static Username & Password + // Hardcoded Username and Password + Username string `yaml:"turn_username"` + Password string `yaml:"turn_password"` + } + // The internal addresses the components will listen on. // These should not be exposed externally as they expose metrics and debugging APIs. Listen struct { @@ -341,10 +361,20 @@ func (config *Dendrite) check(monolithic bool) error { } } + checkValidDuration := func(key, value string) { + if _, err := time.ParseDuration(config.TURN.UserLifetime); err != nil { + problems = append(problems, fmt.Sprintf("invalid duration for config key %q: %s", key, value)) + } + } + checkNotEmpty("matrix.server_name", string(config.Matrix.ServerName)) checkNotEmpty("matrix.private_key", string(config.Matrix.PrivateKeyPath)) checkNotZero("matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths))) + if config.TURN.UserLifetime != "" { + checkValidDuration("turn.turn_user_lifetime", config.TURN.UserLifetime) + } + checkNotEmpty("media.base_path", string(config.Media.BasePath)) checkPositive("media.max_file_size_bytes", int64(*config.Media.MaxFileSizeBytes)) checkPositive("media.max_thumbnail_generators", int64(config.Media.MaxThumbnailGenerators)) From bf855a7e5c79aad38d9bb402443c4c623e7941ea Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 13 Nov 2017 18:39:09 +0000 Subject: [PATCH 13/29] Factor out keyring initialisation (#329) Take the keyring init from the client-api-server and the monolith out to a shared file --- .../cmd/dendrite-client-api-server/main.go | 8 +---- .../cmd/dendrite-monolith-server/main.go | 8 +---- .../dendrite/common/keydb/keyring.go | 32 +++++++++++++++++++ 3 files changed, 34 insertions(+), 14 deletions(-) create mode 100644 src/github.com/matrix-org/dendrite/common/keydb/keyring.go diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-client-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-client-api-server/main.go index a7aa1af99..929fd3b5a 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-client-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-client-api-server/main.go @@ -98,13 +98,7 @@ func main() { log.Panicf("Failed to setup key database(%q): %s", cfg.Database.ServerKey, err.Error()) } - keyRing := gomatrixserverlib.KeyRing{ - KeyFetchers: []gomatrixserverlib.KeyFetcher{ - // TODO: Use perspective key fetchers for production. - &gomatrixserverlib.DirectKeyFetcher{Client: federation.Client}, - }, - KeyDatabase: keyDB, - } + keyRing := keydb.CreateKeyRing(federation.Client, keyDB) kafkaConsumer, err := sarama.NewConsumer(cfg.Kafka.Addresses, nil) if err != nil { diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go index b18d0360c..3b5975763 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go @@ -194,13 +194,7 @@ func (m *monolith) setupFederation() { m.cfg.Matrix.ServerName, m.cfg.Matrix.KeyID, m.cfg.Matrix.PrivateKey, ) - m.keyRing = gomatrixserverlib.KeyRing{ - KeyFetchers: []gomatrixserverlib.KeyFetcher{ - // TODO: Use perspective key fetchers for production. - &gomatrixserverlib.DirectKeyFetcher{Client: m.federation.Client}, - }, - KeyDatabase: m.keyDB, - } + m.keyRing = keydb.CreateKeyRing(m.federation.Client, m.keyDB) } func (m *monolith) setupKafka() { diff --git a/src/github.com/matrix-org/dendrite/common/keydb/keyring.go b/src/github.com/matrix-org/dendrite/common/keydb/keyring.go new file mode 100644 index 000000000..1b20f7816 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/common/keydb/keyring.go @@ -0,0 +1,32 @@ +// Copyright 2017 New Vector Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keydb + +import "github.com/matrix-org/gomatrixserverlib" + +// CreateKeyRing creates and configures a KeyRing object. +// +// It creates the necessary key fetchers and collects them into a KeyRing +// backed by the given KeyDatabase. +func CreateKeyRing(client gomatrixserverlib.Client, + keyDB gomatrixserverlib.KeyDatabase) gomatrixserverlib.KeyRing { + return gomatrixserverlib.KeyRing{ + KeyFetchers: []gomatrixserverlib.KeyFetcher{ + // TODO: Use perspective key fetchers for production. + &gomatrixserverlib.DirectKeyFetcher{Client: client}, + }, + KeyDatabase: keyDB, + } +} From 8720570bb0b229530b6b41d8287f3d7a478fdb95 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 14 Nov 2017 01:56:23 -0800 Subject: [PATCH 14/29] Check for existing filter before inserting a new one (#318) Signed-off-by: Andrew (anoa) --- .../auth/storage/accounts/filter_table.go | 44 ++++++++++++++++--- .../auth/storage/accounts/storage.go | 13 +++--- .../dendrite/clientapi/routing/filter.go | 4 +- .../dendrite/clientapi/routing/register.go | 2 +- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go index c50cd1fd9..9e3b7d6e6 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go @@ -17,6 +17,8 @@ package accounts import ( "context" "database/sql" + + "github.com/matrix-org/gomatrixserverlib" ) const filterSchema = ` @@ -38,12 +40,16 @@ CREATE INDEX IF NOT EXISTS account_filter_localpart ON account_filter(localpart) const selectFilterSQL = "" + "SELECT filter FROM account_filter WHERE localpart = $1 AND id = $2" +const selectFilterIDByContentSQL = "" + + "SELECT id FROM account_filter WHERE localpart = $1 AND filter = $2" + const insertFilterSQL = "" + "INSERT INTO account_filter (filter, id, localpart) VALUES ($1, DEFAULT, $2) RETURNING id" type filterStatements struct { - selectFilterStmt *sql.Stmt - insertFilterStmt *sql.Stmt + selectFilterStmt *sql.Stmt + selectFilterIDByContentStmt *sql.Stmt + insertFilterStmt *sql.Stmt } func (s *filterStatements) prepare(db *sql.DB) (err error) { @@ -54,6 +60,9 @@ func (s *filterStatements) prepare(db *sql.DB) (err error) { if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil { return } + if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil { + return + } if s.insertFilterStmt, err = db.Prepare(insertFilterSQL); err != nil { return } @@ -62,14 +71,37 @@ func (s *filterStatements) prepare(db *sql.DB) (err error) { func (s *filterStatements) selectFilter( ctx context.Context, localpart string, filterID string, -) (filter string, err error) { +) (filter []byte, err error) { err = s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filter) return } func (s *filterStatements) insertFilter( - ctx context.Context, filter string, localpart string, -) (pos string, err error) { - err = s.insertFilterStmt.QueryRowContext(ctx, filter, localpart).Scan(&pos) + ctx context.Context, filter []byte, localpart string, +) (filterID string, err error) { + var existingFilterID string + + // This can result in a race condition when two clients try to insert the + // same filter and localpart at the same time, however this is not a + // problem as both calls will result in the same filterID + filterJSON, err := gomatrixserverlib.CanonicalJSON(filter) + if err != nil { + return "", err + } + + // Check if filter already exists in the database + err = s.selectFilterIDByContentStmt.QueryRowContext(ctx, + localpart, filterJSON).Scan(&existingFilterID) + if err != nil { + return "", err + } + // If it does, return the existing ID + if len(existingFilterID) != 0 { + return existingFilterID, err + } + + // Otherwise insert the filter and return the new ID + err = s.insertFilterStmt.QueryRowContext(ctx, filterJSON, localpart). + Scan(&filterID) return } diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/storage.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/storage.go index 33fbbd86d..d5712eb50 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/storage.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/storage.go @@ -321,22 +321,25 @@ func (d *Database) GetThreePIDsForLocalpart( } // GetFilter looks up the filter associated with a given local user and filter ID. -// Returns an error if no such filter exists or if there was an error taling to the database. +// Returns a filter represented as a byte slice. Otherwise returns an error if +// no such filter exists or if there was an error talking to the database. func (d *Database) GetFilter( ctx context.Context, localpart string, filterID string, -) (string, error) { +) ([]byte, error) { return d.filter.selectFilter(ctx, localpart, filterID) } // PutFilter puts the passed filter into the database. -// Returns an error if something goes wrong. +// Returns the filterID as a string. Otherwise returns an error if something +// goes wrong. func (d *Database) PutFilter( - ctx context.Context, localpart, filter string, + ctx context.Context, localpart string, filter []byte, ) (string, error) { return d.filter.insertFilter(ctx, filter, localpart) } -// CheckAccountAvailability checks if the username/localpart is already present in the database. +// CheckAccountAvailability checks if the username/localpart is already present +// in the database. // If the DB returns sql.ErrNoRows the Localpart isn't taken. func (d *Database) CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) { _, err := d.accounts.selectAccountByLocalpart(ctx, localpart) diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/filter.go b/src/github.com/matrix-org/dendrite/clientapi/routing/filter.go index 3c623147a..4b84e293d 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/filter.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/filter.go @@ -60,7 +60,7 @@ func GetFilter( } } filter := gomatrix.Filter{} - err = json.Unmarshal([]byte(res), &filter) + err = json.Unmarshal(res, &filter) if err != nil { httputil.LogThenError(req, err) } @@ -111,7 +111,7 @@ func PutFilter( } } - filterID, err := accountDB.PutFilter(req.Context(), localpart, string(filterArray)) + filterID, err := accountDB.PutFilter(req.Context(), localpart, filterArray) if err != nil { return httputil.LogThenError(req, err) } diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/register.go b/src/github.com/matrix-org/dendrite/clientapi/routing/register.go index 92c9b427e..29e25764b 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/register.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/register.go @@ -28,7 +28,6 @@ import ( "github.com/matrix-org/dendrite/common/config" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" @@ -37,6 +36,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" ) const ( From bad701c703cb97c8352fea671e506f47c063a91c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20T=C3=B6tterman?= Date: Tue, 14 Nov 2017 11:59:02 +0200 Subject: [PATCH 15/29] Add device display names (#319) --- .../auth/storage/devices/devices_table.go | 32 +++++++--- .../clientapi/auth/storage/devices/storage.go | 15 ++++- .../dendrite/clientapi/routing/device.go | 58 +++++++++++++++++++ .../dendrite/clientapi/routing/login.go | 7 ++- .../dendrite/clientapi/routing/register.go | 13 +++-- .../dendrite/clientapi/routing/routing.go | 7 +++ .../dendrite/cmd/create-account/main.go | 2 +- 7 files changed, 115 insertions(+), 19 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go index a614eb541..903471afe 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go @@ -40,7 +40,9 @@ CREATE TABLE IF NOT EXISTS device_devices ( -- migration to different domain names easier. localpart TEXT NOT NULL, -- When this devices was first recognised on the network, as a unix timestamp (ms resolution). - created_ts BIGINT NOT NULL + created_ts BIGINT NOT NULL, + -- The display name, human friendlier than device_id and updatable + display_name TEXT -- TODO: device keys, device display names, last used ts and IP address?, token restrictions (if 3rd-party OAuth app) ); @@ -49,16 +51,19 @@ CREATE UNIQUE INDEX IF NOT EXISTS device_localpart_id_idx ON device_devices(loca ` const insertDeviceSQL = "" + - "INSERT INTO device_devices(device_id, localpart, access_token, created_ts) VALUES ($1, $2, $3, $4)" + "INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5)" const selectDeviceByTokenSQL = "" + - "SELECT device_id, localpart FROM device_devices WHERE access_token = $1" + "SELECT device_id, localpart, display_name FROM device_devices WHERE access_token = $1" const selectDeviceByIDSQL = "" + - "SELECT created_ts FROM device_devices WHERE localpart = $1 and device_id = $2" + "SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2" const selectDevicesByLocalpartSQL = "" + - "SELECT device_id FROM device_devices WHERE localpart = $1" + "SELECT device_id, display_name FROM device_devices WHERE localpart = $1" + +const updateDeviceNameSQL = "" + + "UPDATE device_devices SET display_name = $1 WHERE localpart = $2 AND device_id = $3" const deleteDeviceSQL = "" + "DELETE FROM device_devices WHERE device_id = $1 AND localpart = $2" @@ -66,13 +71,12 @@ const deleteDeviceSQL = "" + const deleteDevicesByLocalpartSQL = "" + "DELETE FROM device_devices WHERE localpart = $1" -// TODO: List devices? - type devicesStatements struct { insertDeviceStmt *sql.Stmt selectDeviceByTokenStmt *sql.Stmt selectDeviceByIDStmt *sql.Stmt selectDevicesByLocalpartStmt *sql.Stmt + updateDeviceNameStmt *sql.Stmt deleteDeviceStmt *sql.Stmt deleteDevicesByLocalpartStmt *sql.Stmt serverName gomatrixserverlib.ServerName @@ -95,6 +99,9 @@ func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerN if s.selectDevicesByLocalpartStmt, err = db.Prepare(selectDevicesByLocalpartSQL); err != nil { return } + if s.updateDeviceNameStmt, err = db.Prepare(updateDeviceNameSQL); err != nil { + return + } if s.deleteDeviceStmt, err = db.Prepare(deleteDeviceSQL); err != nil { return } @@ -110,10 +117,11 @@ func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerN // Returns the device on success. func (s *devicesStatements) insertDevice( ctx context.Context, txn *sql.Tx, id, localpart, accessToken string, + displayName *string, ) (*authtypes.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 stmt := common.TxStmt(txn, s.insertDeviceStmt) - if _, err := stmt.ExecContext(ctx, id, localpart, accessToken, createdTimeMS); err != nil { + if _, err := stmt.ExecContext(ctx, id, localpart, accessToken, createdTimeMS, displayName); err != nil { return nil, err } return &authtypes.Device{ @@ -139,6 +147,14 @@ func (s *devicesStatements) deleteDevicesByLocalpart( return err } +func (s *devicesStatements) updateDeviceName( + ctx context.Context, txn *sql.Tx, localpart, deviceID string, displayName *string, +) error { + stmt := common.TxStmt(txn, s.updateDeviceNameStmt) + _, err := stmt.ExecContext(ctx, displayName, localpart, deviceID) + return err +} + func (s *devicesStatements) selectDeviceByToken( ctx context.Context, accessToken string, ) (*authtypes.Device, error) { diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go index dd98bb609..6ac475a66 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go @@ -75,6 +75,7 @@ func (d *Database) GetDevicesByLocalpart( // Returns the device on success. func (d *Database) CreateDevice( ctx context.Context, localpart string, deviceID *string, accessToken string, + displayName *string, ) (dev *authtypes.Device, returnErr error) { if deviceID != nil { returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { @@ -84,7 +85,7 @@ func (d *Database) CreateDevice( return err } - dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken) + dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken, displayName) return err }) } else { @@ -99,7 +100,7 @@ func (d *Database) CreateDevice( returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { var err error - dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken) + dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) return err }) if returnErr == nil { @@ -110,6 +111,16 @@ func (d *Database) CreateDevice( return } +// UpdateDevice updates the given device with the display name. +// Returns SQL error if there are problems and nil on success. +func (d *Database) UpdateDevice( + ctx context.Context, localpart, deviceID string, displayName *string, +) error { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return d.devices.updateDeviceName(ctx, txn, localpart, deviceID, displayName) + }) +} + // RemoveDevice revokes a device by deleting the entry in the database // matching with the given device ID and user ID localpart // If the device doesn't exist, it will not return an error diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/device.go b/src/github.com/matrix-org/dendrite/clientapi/routing/device.go index 9cb63bac9..86e393be1 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/device.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/device.go @@ -16,6 +16,7 @@ package routing import ( "database/sql" + "encoding/json" "net/http" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -35,6 +36,10 @@ type devicesJSON struct { Devices []deviceJSON `json:"devices"` } +type deviceUpdateJSON struct { + DisplayName *string `json:"display_name"` +} + // GetDeviceByID handles /device/{deviceID} func GetDeviceByID( req *http.Request, deviceDB *devices.Database, device *authtypes.Device, @@ -95,3 +100,56 @@ func GetDevicesByLocalpart( JSON: res, } } + +// UpdateDeviceByID handles PUT on /devices/{deviceID} +func UpdateDeviceByID( + req *http.Request, deviceDB *devices.Database, device *authtypes.Device, + deviceID string, +) util.JSONResponse { + if req.Method != "PUT" { + return util.JSONResponse{ + Code: 405, + JSON: jsonerror.NotFound("Bad Method"), + } + } + + localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) + if err != nil { + return httputil.LogThenError(req, err) + } + + ctx := req.Context() + dev, err := deviceDB.GetDeviceByID(ctx, localpart, deviceID) + if err == sql.ErrNoRows { + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound("Unknown device"), + } + } else if err != nil { + return httputil.LogThenError(req, err) + } + + if dev.UserID != device.UserID { + return util.JSONResponse{ + Code: 403, + JSON: jsonerror.Forbidden("device not owned by current user"), + } + } + + defer req.Body.Close() // nolint: errcheck + + payload := deviceUpdateJSON{} + + if err := json.NewDecoder(req.Body).Decode(&payload); err != nil { + return httputil.LogThenError(req, err) + } + + if err := deviceDB.UpdateDevice(ctx, localpart, deviceID, payload.DisplayName); err != nil { + return httputil.LogThenError(req, err) + } + + return util.JSONResponse{ + Code: 200, + JSON: struct{}{}, + } +} diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/login.go b/src/github.com/matrix-org/dendrite/clientapi/routing/login.go index 2fa44d6d6..56c67b77d 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/login.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/login.go @@ -38,8 +38,9 @@ type flow struct { } type passwordRequest struct { - User string `json:"user"` - Password string `json:"password"` + User string `json:"user"` + Password string `json:"password"` + InitialDisplayName *string `json:"initial_device_display_name"` } type loginResponse struct { @@ -119,7 +120,7 @@ func Login( // TODO: Use the device ID in the request dev, err := deviceDB.CreateDevice( - req.Context(), acc.Localpart, nil, token, + req.Context(), acc.Localpart, nil, token, r.InitialDisplayName, ) if err != nil { return util.JSONResponse{ diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/register.go b/src/github.com/matrix-org/dendrite/clientapi/routing/register.go index 29e25764b..875ceb049 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/register.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/register.go @@ -60,6 +60,8 @@ type registerRequest struct { Admin bool `json:"admin"` // user-interactive auth params Auth authDict `json:"auth"` + + InitialDisplayName *string `json:"initial_device_display_name"` } type authDict struct { @@ -210,10 +212,10 @@ func Register( return util.MessageResponse(403, "HMAC incorrect") } - return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password) + return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, r.InitialDisplayName) case authtypes.LoginTypeDummy: // there is nothing to do - return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password) + return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, r.InitialDisplayName) default: return util.JSONResponse{ Code: 501, @@ -270,10 +272,10 @@ func LegacyRegister( return util.MessageResponse(403, "HMAC incorrect") } - return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password) + return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, nil) case authtypes.LoginTypeDummy: // there is nothing to do - return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password) + return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, nil) default: return util.JSONResponse{ Code: 501, @@ -287,6 +289,7 @@ func completeRegistration( accountDB *accounts.Database, deviceDB *devices.Database, username, password string, + displayName *string, ) util.JSONResponse { if username == "" { return util.JSONResponse{ @@ -318,7 +321,7 @@ func completeRegistration( } // // TODO: Use the device ID in the request. - dev, err := deviceDB.CreateDevice(ctx, username, nil, token) + dev, err := deviceDB.CreateDevice(ctx, username, nil, token, displayName) if err != nil { return util.JSONResponse{ Code: 500, diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go b/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go index 87f52ad09..9e9fea5a6 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go @@ -364,6 +364,13 @@ func Setup( }), ).Methods("GET") + r0mux.Handle("/devices/{deviceID}", + common.MakeAuthAPI("device_data", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars := mux.Vars(req) + return UpdateDeviceByID(req, deviceDB, device, vars["deviceID"]) + }), + ).Methods("PUT", "OPTIONS") + // Stub implementations for sytest r0mux.Handle("/events", common.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse { diff --git a/src/github.com/matrix-org/dendrite/cmd/create-account/main.go b/src/github.com/matrix-org/dendrite/cmd/create-account/main.go index 3d5c35878..7914a6266 100644 --- a/src/github.com/matrix-org/dendrite/cmd/create-account/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/create-account/main.go @@ -87,7 +87,7 @@ func main() { } device, err := deviceDB.CreateDevice( - context.Background(), *username, nil, *accessToken, + context.Background(), *username, nil, *accessToken, nil, ) if err != nil { fmt.Println(err.Error()) From dc782ec399372b2c63623555dcb82f0641bfdd3c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Nov 2017 14:58:27 +0000 Subject: [PATCH 16/29] Split travis into multiple jobs (#322) The motivation for this is to make it easier to see whether a travis failure is due to linting, unit tests or integration test failures, without having to look in the logs. It also means that each job is independent, so if e.g. the linting fails then the unit tests will still be run. --- .travis.yml | 14 ++++----- scripts/build-test-lint.sh | 3 -- scripts/find-lint.sh | 2 +- scripts/travis-test.sh | 58 ++++++++++++++++++++++++++++---------- 4 files changed, 50 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index dd7359af9..2bbc65655 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,12 @@ language: go go: - 1.8 + - 1.9 + +env: +- TEST_SUITE="lint" +- TEST_SUITE="unit-test" +- TEST_SUITE="integ-test" sudo: false @@ -8,9 +14,6 @@ sudo: false dist: trusty addons: - apt: - packages: - - openssl postgresql: "9.5" services: @@ -19,12 +22,7 @@ services: install: - go get github.com/constabulary/gb/... -# Generate a self-signed X.509 certificate for TLS. -before_script: - - openssl req -x509 -newkey rsa:4096 -keyout server.key -out server.crt -days 365 -nodes -subj /CN=localhost - script: - - ./scripts/install-local-kafka.sh - ./scripts/travis-test.sh notifications: diff --git a/scripts/build-test-lint.sh b/scripts/build-test-lint.sh index cf1f37b78..16078173d 100755 --- a/scripts/build-test-lint.sh +++ b/scripts/build-test-lint.sh @@ -19,8 +19,5 @@ go build github.com/matrix-org/dendrite/cmd/... ./scripts/find-lint.sh -echo "Double checking spelling..." -misspell -error src *.md - echo "Testing..." gb test diff --git a/scripts/find-lint.sh b/scripts/find-lint.sh index 81ca806b0..e2df519c5 100755 --- a/scripts/find-lint.sh +++ b/scripts/find-lint.sh @@ -31,7 +31,7 @@ then args="$args --enable-gc" fi echo "Installing lint search engine..." -go install github.com/alecthomas/gometalinter/ +gb build github.com/alecthomas/gometalinter/ gometalinter --config=linter.json ./... --install echo "Looking for lint..." diff --git a/scripts/travis-test.sh b/scripts/travis-test.sh index 5851c3b0f..e472760a6 100755 --- a/scripts/travis-test.sh +++ b/scripts/travis-test.sh @@ -1,6 +1,10 @@ #! /bin/bash # The entry point for travis tests +# +# TEST_SUITE env var can be set to "lint", "unit-test" or "integ-test", in +# which case only the linting, unit tests or integration tests will be run +# respectively. If not specified or null all tests are run. set -eu @@ -8,20 +12,44 @@ set -eu export GOGC=400 export DENDRITE_LINT_DISABLE_GC=1 -# Check that the servers build (this is done explicitly because `gb build` can silently fail (exit 0) and then we'd test a stale binary) -gb build github.com/matrix-org/dendrite/cmd/dendrite-room-server -gb build github.com/matrix-org/dendrite/cmd/roomserver-integration-tests -gb build github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server -gb build github.com/matrix-org/dendrite/cmd/syncserver-integration-tests -gb build github.com/matrix-org/dendrite/cmd/create-account -gb build github.com/matrix-org/dendrite/cmd/dendrite-media-api-server -gb build github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests -gb build github.com/matrix-org/dendrite/cmd/client-api-proxy +export GOPATH="$(pwd):$(pwd)/vendor" +export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin" -# Run unit tests and linters -./scripts/build-test-lint.sh +if [ "${TEST_SUITE:-lint}" == "lint" ]; then + ./scripts/find-lint.sh +fi -# Run the integration tests -bin/roomserver-integration-tests -bin/syncserver-integration-tests -bin/mediaapi-integration-tests +if [ "${TEST_SUITE:-unit-test}" == "unit-test" ]; then + gb test +fi + +if [ "${TEST_SUITE:-integ-test}" == "integ-test" ]; then + gb build + + # Check that all the packages can build. + # When `go build` is given multiple packages it won't output anything, and just + # checks that everything builds. This seems to do a better job of handling + # missing imports than `gb build` does. + go build github.com/matrix-org/dendrite/cmd/... + + # Check that the servers build (this is done explicitly because `gb build` can silently fail (exit 0) and then we'd test a stale binary) + gb build github.com/matrix-org/dendrite/cmd/dendrite-room-server + gb build github.com/matrix-org/dendrite/cmd/roomserver-integration-tests + gb build github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server + gb build github.com/matrix-org/dendrite/cmd/syncserver-integration-tests + gb build github.com/matrix-org/dendrite/cmd/create-account + gb build github.com/matrix-org/dendrite/cmd/dendrite-media-api-server + gb build github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests + gb build github.com/matrix-org/dendrite/cmd/client-api-proxy + + # Create necessary certificates and keys to run dendrite + echo "Generating certs..." + time openssl req -x509 -newkey rsa:512 -keyout server.key -out server.crt -days 365 -nodes -subj /CN=localhost + echo "Installing kafka..." + time ./scripts/install-local-kafka.sh + + # Run the integration tests + bin/roomserver-integration-tests + bin/syncserver-integration-tests + bin/mediaapi-integration-tests +fi From 8fff0e887c1a3a655d03a40011c79adbd6ea3694 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 15 Nov 2017 10:25:48 +0000 Subject: [PATCH 17/29] Update gometalinter config (#331) * Update gometalinter config gometalinter now uses `maligned` instead of `aligncheck` (https://github.com/alecthomas/gometalinter/pull/367), so we need to update our config accordingly. * Update gometalinter * Disable gotype linter gotype does not seem to play nicely with the gb vendor directory. In particular, it wants each of our dependencies to be built and installed (see https://github.com/golang/go/issues/10969), but (empirically) it will not accept them being installed in `pkg` but insists on them being in `vendor/pkg`. This presents a problem because `gb build` builds the packages into `pkg` (which doesn't seem entirely unreasonable since `.` comes before `vendor` in `$GOPATH`). `go install github.com/x/y` does install in `vendor/pkg` but requires us to know the name of each package. The general conclusion of https://github.com/alecthomas/gometalinter/issues/91 seems to have been that the easiest thing to do is to disable `gotype` for now. * Fix `unparam` lint * Fix goshadow lint --- linter-fast.json | 1 - linter.json | 3 +- .../dendrite/clientapi/routing/device.go | 4 +- .../dendrite/clientapi/routing/memberships.go | 2 +- .../cmd/mediaapi-integration-tests/main.go | 10 +- .../dendrite/federationapi/routing/events.go | 6 +- .../federationapi/routing/threepid.go | 2 +- .../mediaapi/thumbnailer/thumbnailer.go | 4 +- vendor/manifest | 2 +- .../alecthomas/gometalinter/README.md | 58 +- .../github.com/alexkohler/nakedret/LICENSE | 21 + .../github.com/alexkohler/nakedret/import.go | 310 ++++++++ .../alexkohler/nakedret/nakedret.go | 213 ++++++ .../src/github.com/dnephin/govet/LICENSE | 27 + .../src/github.com/dnephin/govet/asmdecl.go | 682 ++++++++++++++++++ .../src/github.com/dnephin/govet/assign.go | 49 ++ .../src/github.com/dnephin/govet/atomic.go | 69 ++ .../src/github.com/dnephin/govet/bool.go | 186 +++++ .../src/github.com/dnephin/govet/buildtag.go | 91 +++ .../src/github.com/dnephin/govet/cgo.go | 130 ++++ .../src/github.com/dnephin/govet/composite.go | 82 +++ .../src/github.com/dnephin/govet/copylock.go | 239 ++++++ .../src/github.com/dnephin/govet/deadcode.go | 298 ++++++++ .../src/github.com/dnephin/govet/doc.go | 205 ++++++ .../dnephin/govet/internal/cfg/builder.go | 512 +++++++++++++ .../dnephin/govet/internal/cfg/cfg.go | 142 ++++ .../govet/internal/whitelist/whitelist.go | 28 + .../github.com/dnephin/govet/lostcancel.go | 318 ++++++++ .../src/github.com/dnephin/govet/main.go | 504 +++++++++++++ .../src/github.com/dnephin/govet/method.go | 182 +++++ .../src/github.com/dnephin/govet/nilfunc.go | 67 ++ .../src/github.com/dnephin/govet/print.go | 650 +++++++++++++++++ .../src/github.com/dnephin/govet/rangeloop.go | 74 ++ .../src/github.com/dnephin/govet/shadow.go | 246 +++++++ .../src/github.com/dnephin/govet/shift.go | 82 +++ .../src/github.com/dnephin/govet/structtag.go | 122 ++++ .../src/github.com/dnephin/govet/tests.go | 187 +++++ .../src/github.com/dnephin/govet/types.go | 281 ++++++++ .../src/github.com/dnephin/govet/unsafeptr.go | 97 +++ .../src/github.com/dnephin/govet/unused.go | 93 +++ .../kisielk/gotool/{go16.go => go16-18.go} | 2 +- .../kisielk/gotool/internal/load/path.go | 27 + .../kisielk/gotool/internal/load/pkg.go | 25 + .../kisielk/gotool/internal/load/search.go | 354 +++++++++ .../src/github.com/kisielk/gotool/match.go | 284 +------- .../src/github.com/kisielk/gotool/match18.go | 317 ++++++++ .../src/github.com/mdempsky/maligned/LICENSE | 27 + .../github.com/mdempsky/maligned/maligned.go | 229 ++++++ .../src/github.com/mvdan/unparam/main.go | 26 - .../x/tools/container/intsets/sparse.go | 480 +++++++----- .../x/tools/go/ast/astutil/imports.go | 26 +- .../x/tools/imports/fastwalk_dirent_ino.go | 3 +- .../x/tools/imports/fastwalk_unix.go | 3 +- .../src/golang.org/x/tools/imports/fix.go | 2 +- .../golang.org/x/tools/imports/mkstdlib.go | 1 + .../src/golang.org/x/tools/imports/zstdlib.go | 72 ++ .../honnef.co/go/tools/functions/functions.go | 8 + .../src/honnef.co/go/tools/simple/lint.go | 390 +++++----- .../honnef.co/go/tools/staticcheck/lint.go | 83 +-- .../gometalinter/_linters/src/manifest | 140 ++-- .../mvdan => mvdan.cc}/interfacer/LICENSE | 0 .../interfacer/check}/cache.go | 2 +- .../interfacer/check}/check.go | 5 +- .../interfacer/check}/types.go | 2 +- .../cmd => mvdan.cc}/interfacer/main.go | 6 +- .../mvdan => mvdan.cc}/lint/LICENSE | 0 .../lint/cmd/metalint/main.go | 8 +- .../mvdan => mvdan.cc}/lint/lint.go | 2 +- .../mvdan => mvdan.cc}/unparam/LICENSE | 0 .../mvdan => mvdan.cc}/unparam/check/check.go | 188 ++++- .../_linters/src/mvdan.cc/unparam/main.go | 33 + .../alecthomas/gometalinter/aggregate.go | 32 +- .../alecthomas/gometalinter/checkstyle.go | 2 +- .../alecthomas/gometalinter/config.go | 18 +- .../alecthomas/gometalinter/config_test.go | 21 + .../alecthomas/gometalinter/directives.go | 62 ++ .../gometalinter/directives_test.go | 41 ++ .../alecthomas/gometalinter/execute.go | 133 +--- .../alecthomas/gometalinter/execute_test.go | 76 +- .../alecthomas/gometalinter/issue.go | 114 +++ .../alecthomas/gometalinter/issue_test.go | 39 + .../alecthomas/gometalinter/linters.go | 163 +++-- .../alecthomas/gometalinter/linters_test.go | 45 +- .../alecthomas/gometalinter/main.go | 12 +- .../alecthomas/gometalinter/partition.go | 44 +- .../alecthomas/gometalinter/partition_test.go | 21 +- .../regressiontests/aligncheck_test.go | 20 - .../gometalinter/regressiontests/dupl_test.go | 2 +- .../regressiontests/gosimple_test.go | 2 +- .../regressiontests/gotype_test.go | 54 +- .../regressiontests/maligned_test.go | 20 + .../regressiontests/nakedret_test.go | 29 + .../gometalinter/regressiontests/support.go | 72 +- .../regressiontests/unparam_test.go | 10 +- .../gometalinter/regressiontests/vet_test.go | 47 +- 95 files changed, 8927 insertions(+), 1176 deletions(-) create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/alexkohler/nakedret/LICENSE create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/alexkohler/nakedret/import.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/alexkohler/nakedret/nakedret.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/LICENSE create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/asmdecl.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/assign.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/atomic.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/bool.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/buildtag.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/cgo.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/composite.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/copylock.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/deadcode.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/doc.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/internal/cfg/builder.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/internal/cfg/cfg.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/internal/whitelist/whitelist.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/lostcancel.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/main.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/method.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/nilfunc.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/print.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/rangeloop.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/shadow.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/shift.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/structtag.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/tests.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/types.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/unsafeptr.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/unused.go rename vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/{go16.go => go16-18.go} (90%) create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/internal/load/path.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/internal/load/pkg.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/internal/load/search.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/match18.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mdempsky/maligned/LICENSE create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mdempsky/maligned/maligned.go delete mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/unparam/main.go rename vendor/src/github.com/alecthomas/gometalinter/_linters/src/{github.com/mvdan => mvdan.cc}/interfacer/LICENSE (100%) rename vendor/src/github.com/alecthomas/gometalinter/_linters/src/{github.com/mvdan/interfacer => mvdan.cc/interfacer/check}/cache.go (98%) rename vendor/src/github.com/alecthomas/gometalinter/_linters/src/{github.com/mvdan/interfacer => mvdan.cc/interfacer/check}/check.go (99%) rename vendor/src/github.com/alecthomas/gometalinter/_linters/src/{github.com/mvdan/interfacer => mvdan.cc/interfacer/check}/types.go (99%) rename vendor/src/github.com/alecthomas/gometalinter/_linters/src/{github.com/mvdan/interfacer/cmd => mvdan.cc}/interfacer/main.go (75%) rename vendor/src/github.com/alecthomas/gometalinter/_linters/src/{github.com/mvdan => mvdan.cc}/lint/LICENSE (100%) rename vendor/src/github.com/alecthomas/gometalinter/_linters/src/{github.com/mvdan => mvdan.cc}/lint/cmd/metalint/main.go (92%) rename vendor/src/github.com/alecthomas/gometalinter/_linters/src/{github.com/mvdan => mvdan.cc}/lint/lint.go (93%) rename vendor/src/github.com/alecthomas/gometalinter/_linters/src/{github.com/mvdan => mvdan.cc}/unparam/LICENSE (100%) rename vendor/src/github.com/alecthomas/gometalinter/_linters/src/{github.com/mvdan => mvdan.cc}/unparam/check/check.go (62%) create mode 100644 vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/unparam/main.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/config_test.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/issue.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/issue_test.go delete mode 100644 vendor/src/github.com/alecthomas/gometalinter/regressiontests/aligncheck_test.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/regressiontests/maligned_test.go create mode 100644 vendor/src/github.com/alecthomas/gometalinter/regressiontests/nakedret_test.go diff --git a/linter-fast.json b/linter-fast.json index 3825fa615..aa86054e8 100644 --- a/linter-fast.json +++ b/linter-fast.json @@ -4,7 +4,6 @@ "Deadline": "5m", "Enable": [ "vetshadow", - "gotype", "deadcode", "gocyclo", "ineffassign", diff --git a/linter.json b/linter.json index 53308ba51..511c244b6 100644 --- a/linter.json +++ b/linter.json @@ -4,13 +4,12 @@ "Deadline": "5m", "Enable": [ "vetshadow", - "gotype", "deadcode", "gocyclo", "golint", "varcheck", "structcheck", - "aligncheck", + "maligned", "ineffassign", "gas", "misspell", diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/device.go b/src/github.com/matrix-org/dendrite/clientapi/routing/device.go index 86e393be1..75df2e6e4 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/device.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/device.go @@ -80,7 +80,7 @@ func GetDevicesByLocalpart( } ctx := req.Context() - devices, err := deviceDB.GetDevicesByLocalpart(ctx, localpart) + deviceList, err := deviceDB.GetDevicesByLocalpart(ctx, localpart) if err != nil { return httputil.LogThenError(req, err) @@ -88,7 +88,7 @@ func GetDevicesByLocalpart( res := devicesJSON{} - for _, dev := range devices { + for _, dev := range deviceList { res.Devices = append(res.Devices, deviceJSON{ DeviceID: dev.ID, UserID: dev.UserID, diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/memberships.go b/src/github.com/matrix-org/dendrite/clientapi/routing/memberships.go index a4b605313..fde278a5b 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/memberships.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/memberships.go @@ -33,7 +33,7 @@ type response struct { // GetMemberships implements GET /rooms/{roomId}/members func GetMemberships( req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool, - cfg config.Dendrite, + _ config.Dendrite, queryAPI api.RoomserverQueryAPI, ) util.JSONResponse { queryReq := api.QueryMembershipsForRoomRequest{ diff --git a/src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go b/src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go index 722cb80f3..d8b457d6b 100644 --- a/src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go @@ -75,7 +75,7 @@ var timeout time.Duration var port = 10000 -func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error, string, *exec.Cmd, chan error, string, string) { +func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error, *exec.Cmd, string, string) { dir, err := ioutil.TempDir("", serverType+"-server-test"+suffix) if err != nil { panic(err) @@ -107,7 +107,7 @@ func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error testDatabaseName + suffix, } - proxyCmd, proxyCmdChan := test.StartProxy(proxyAddr, cfg) + proxyCmd, _ := test.StartProxy(proxyAddr, cfg) test.InitDatabase( postgresDatabase, @@ -121,7 +121,7 @@ func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error ) fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, cfg.Listen.MediaAPI, dir) - return cmd, cmdChan, string(cfg.Listen.MediaAPI), proxyCmd, proxyCmdChan, proxyAddr, dir + return cmd, cmdChan, proxyCmd, proxyAddr, dir } func cleanUpServer(cmd *exec.Cmd, dir string) { @@ -145,7 +145,7 @@ func main() { } // create server1 with only pre-generated thumbnails allowed - server1Cmd, server1CmdChan, _, server1ProxyCmd, _, server1ProxyAddr, server1Dir := startMediaAPI("1", false) + server1Cmd, server1CmdChan, server1ProxyCmd, server1ProxyAddr, server1Dir := startMediaAPI("1", false) defer cleanUpServer(server1Cmd, server1Dir) defer server1ProxyCmd.Process.Kill() // nolint: errcheck testDownload(server1ProxyAddr, server1ProxyAddr, "doesnotexist", 404, server1CmdChan) @@ -162,7 +162,7 @@ func main() { testThumbnail(64, 64, "crop", server1ProxyAddr, server1CmdChan) // create server2 with dynamic thumbnail generation - server2Cmd, server2CmdChan, _, server2ProxyCmd, _, server2ProxyAddr, server2Dir := startMediaAPI("2", true) + server2Cmd, server2CmdChan, server2ProxyCmd, server2ProxyAddr, server2Dir := startMediaAPI("2", true) defer cleanUpServer(server2Cmd, server2Dir) defer server2ProxyCmd.Process.Kill() // nolint: errcheck testDownload(server2ProxyAddr, server2ProxyAddr, "doesnotexist", 404, server2CmdChan) diff --git a/src/github.com/matrix-org/dendrite/federationapi/routing/events.go b/src/github.com/matrix-org/dendrite/federationapi/routing/events.go index 0b2fdcf54..ed1173113 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/routing/events.go +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/events.go @@ -28,10 +28,10 @@ import ( func GetEvent( ctx context.Context, request *gomatrixserverlib.FederationRequest, - cfg config.Dendrite, + _ config.Dendrite, query api.RoomserverQueryAPI, - now time.Time, - keys gomatrixserverlib.KeyRing, + _ time.Time, + _ gomatrixserverlib.KeyRing, eventID string, ) util.JSONResponse { var authResponse api.QueryServerAllowedToSeeEventResponse diff --git a/src/github.com/matrix-org/dendrite/federationapi/routing/threepid.go b/src/github.com/matrix-org/dendrite/federationapi/routing/threepid.go index 47549ca6e..cf261c450 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/routing/threepid.go +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/threepid.go @@ -287,7 +287,7 @@ func buildMembershipEvent( // them responded with an error. func sendToRemoteServer( ctx context.Context, inv invite, - federation *gomatrixserverlib.FederationClient, cfg config.Dendrite, + federation *gomatrixserverlib.FederationClient, _ config.Dendrite, builder gomatrixserverlib.EventBuilder, ) (err error) { remoteServers := make([]gomatrixserverlib.ServerName, 2) diff --git a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go index 3d4b0d4e9..efef03b5f 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go @@ -89,7 +89,7 @@ func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailM } // getActiveThumbnailGeneration checks for active thumbnail generation -func getActiveThumbnailGeneration(dst types.Path, config types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, logger *log.Entry) (isActive bool, busy bool, errorReturn error) { +func getActiveThumbnailGeneration(dst types.Path, _ types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, logger *log.Entry) (isActive bool, busy bool, errorReturn error) { // Check if there is active thumbnail generation. activeThumbnailGeneration.Lock() defer activeThumbnailGeneration.Unlock() @@ -119,7 +119,7 @@ func getActiveThumbnailGeneration(dst types.Path, config types.ThumbnailSize, ac // broadcastGeneration broadcasts that thumbnail generation completed and the error to all waiting goroutines // Note: This should only be called by the owner of the activeThumbnailGenerationResult -func broadcastGeneration(dst types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, config types.ThumbnailSize, errorReturn error, logger *log.Entry) { +func broadcastGeneration(dst types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, _ types.ThumbnailSize, errorReturn error, logger *log.Entry) { activeThumbnailGeneration.Lock() defer activeThumbnailGeneration.Unlock() if activeThumbnailGenerationResult, ok := activeThumbnailGeneration.PathToResult[string(dst)]; ok { diff --git a/vendor/manifest b/vendor/manifest index fb8b376d0..6ee014b4a 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -10,7 +10,7 @@ { "importpath": "github.com/alecthomas/gometalinter", "repository": "https://github.com/alecthomas/gometalinter", - "revision": "5507b26af3204e949ffe50ec08ee73e5847938e1", + "revision": "0262fb20957a4c2d3bb7c834a6a125ae3884a2c6", "branch": "master" }, { diff --git a/vendor/src/github.com/alecthomas/gometalinter/README.md b/vendor/src/github.com/alecthomas/gometalinter/README.md index fc177ad9e..809a60ad9 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/README.md +++ b/vendor/src/github.com/alecthomas/gometalinter/README.md @@ -19,7 +19,6 @@ - [2. Analyse the debug output](#2-analyse-the-debug-output) - [3. Report an issue.](#3-report-an-issue) - [How do I filter issues between two git refs?](#how-do-i-filter-issues-between-two-git-refs) -- [Details](#details) - [Checkstyle XML format](#checkstyle-xml-format) @@ -57,12 +56,13 @@ It is intended for use with editor/IDE integration. - [go vet](https://golang.org/cmd/vet/) - Reports potential errors that otherwise compile. - [go tool vet --shadow](https://golang.org/cmd/vet/#hdr-Shadowed_variables) - Reports variables that may have been unintentionally shadowed. - [gotype](https://golang.org/x/tools/cmd/gotype) - Syntactic and semantic analysis similar to the Go compiler. +- [gotype -x](https://golang.org/x/tools/cmd/gotype) - Syntactic and semantic analysis in external test packages (similar to the Go compiler). - [deadcode](https://github.com/tsenart/deadcode) - Finds unused code. - [gocyclo](https://github.com/alecthomas/gocyclo) - Computes the cyclomatic complexity of functions. - [golint](https://github.com/golang/lint) - Google's (mostly stylistic) linter. - [varcheck](https://github.com/opennota/check) - Find unused global variables and constants. - [structcheck](https://github.com/opennota/check) - Find unused struct fields. -- [aligncheck](https://github.com/opennota/check) - Warn about un-optimally aligned structures. +- [maligned](https://github.com/mdempsky/maligned) - Detect structs that would take less memory if their fields were sorted. - [errcheck](https://github.com/kisielk/errcheck) - Check that error return values are used. - [megacheck](https://github.com/dominikh/go-tools/tree/master/cmd/megacheck) - Run staticcheck, gosimple and unused, sharing work. - [dupl](https://github.com/mibk/dupl) - Reports potentially duplicated code. @@ -81,6 +81,7 @@ Disabled by default (enable with `--enable=`): - [gosimple](https://github.com/dominikh/go-tools/tree/master/cmd/gosimple) - Report simplifications in code. - [lll](https://github.com/walle/lll) - Report long lines (see `--line-length=N`). - [misspell](https://github.com/client9/misspell) - Finds commonly misspelled English words. +- [nakedret](https://github.com/alexkohler/nakedret) - Finds naked returns. - [unparam](https://github.com/mvdan/unparam) - Find unused function parameters. - [unused](https://github.com/dominikh/go-tools/tree/master/cmd/unused) - Find unused variables. - [safesql](https://github.com/stripe/safesql) - Finds potential SQL injection vulnerabilities. @@ -91,14 +92,15 @@ Additional linters can be added through the command line with `--linter=NAME:COM ## Configuration file gometalinter now supports a JSON configuration file which can be loaded via -`--config=`. The format of this file is determined by the Config struct -in `config.go`. +`--config=`. The format of this file is determined by the `Config` struct +in [config.go](https://github.com/alecthomas/gometalinter/blob/master/config.go). The configuration file mostly corresponds to command-line flags, with the following exceptions: - Linters defined in the configuration file will overlay existing definitions, not replace them. - "Enable" defines the exact set of linters that will be enabled (default - linters are disabled). + linters are disabled). `--help` displays the list of default linters with the exact names + you must use. Here is an example configuration file: @@ -108,6 +110,34 @@ Here is an example configuration file: } ``` +### Adding Custom linters + +Linters can be added and customized from the config file using the `Linters` field. +Linters supports the following fields: + +* `Command` - the path to the linter binary and any default arguments +* `Pattern` - a regular expression used to parse the linter output +* `IsFast` - if the linter should be run when the `--fast` flag is used +* `PartitionStrategy` - how paths args should be passed to the linter command: + * `directories` - call the linter once with a list of all the directories + * `files` - call the linter once with a list of all the files + * `packages` - call the linter once with a list of all the package paths + * `files-by-package` - call the linter once per package with a list of the + files in the package. + * `single-directory` - call the linter once per directory + +The config for default linters can be overridden by using the name of the +linter. + +Additional linters can be configured via the command line using the format +`NAME:COMMAND:PATTERN`. + +Example: + +``` +$ gometalinter --linter='vet:go tool vet -printfuncs=Infof,Debugf,Warningf,Errorf:PATH:LINE:MESSAGE' . +``` + ## Installing There are two options for installing gometalinter. @@ -171,7 +201,8 @@ Install all known linters: $ gometalinter --install Installing: structcheck - aligncheck + maligned + nakedret deadcode gocyclo ineffassign @@ -308,21 +339,6 @@ gometalinter |& revgrep master # Show issues between master and HEAD (or gometalinter |& revgrep origin/master # Show issues that haven't been pushed. ``` -## Details - -Additional linters can be configured via the command line: - -``` -$ gometalinter --linter='vet:go tool vet -printfuncs=Infof,Debugf,Warningf,Errorf:PATH:LINE:MESSAGE' . -stutter.go:21:15:warning: error return value not checked (defer a.Close()) (errcheck) -stutter.go:22:15:warning: error return value not checked (defer a.Close()) (errcheck) -stutter.go:27:6:warning: error return value not checked (doit() // test for errcheck) (errcheck) -stutter.go:9::warning: unused global variable unusedGlobal (varcheck) -stutter.go:13::warning: unused struct field MyStruct.Unused (structcheck) -stutter.go:12:6:warning: exported type MyStruct should have comment or be unexported (golint) -stutter.go:16:6:warning: exported type PublicUndocumented should have comment or be unexported (deadcode) -``` - ## Checkstyle XML format `gometalinter` supports [checkstyle](http://checkstyle.sourceforge.net/) diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/alexkohler/nakedret/LICENSE b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/alexkohler/nakedret/LICENSE new file mode 100644 index 000000000..9310fbcff --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/alexkohler/nakedret/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Alex Kohler + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/alexkohler/nakedret/import.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/alexkohler/nakedret/import.go new file mode 100644 index 000000000..e02869163 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/alexkohler/nakedret/import.go @@ -0,0 +1,310 @@ +package main + +/* + +This file holds a direct copy of the import path matching code of +https://github.com/golang/go/blob/master/src/cmd/go/main.go. It can be +replaced when https://golang.org/issue/8768 is resolved. + +It has been updated to follow upstream changes in a few ways. + +*/ + +import ( + "fmt" + "go/build" + "log" + "os" + "path" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +var buildContext = build.Default + +var ( + goroot = filepath.Clean(runtime.GOROOT()) + gorootSrc = filepath.Join(goroot, "src") +) + +// importPathsNoDotExpansion returns the import paths to use for the given +// command line, but it does no ... expansion. +func importPathsNoDotExpansion(args []string) []string { + if len(args) == 0 { + return []string{"."} + } + var out []string + for _, a := range args { + // Arguments are supposed to be import paths, but + // as a courtesy to Windows developers, rewrite \ to / + // in command-line arguments. Handles .\... and so on. + if filepath.Separator == '\\' { + a = strings.Replace(a, `\`, `/`, -1) + } + + // Put argument in canonical form, but preserve leading ./. + if strings.HasPrefix(a, "./") { + a = "./" + path.Clean(a) + if a == "./." { + a = "." + } + } else { + a = path.Clean(a) + } + if a == "all" || a == "std" { + out = append(out, allPackages(a)...) + continue + } + out = append(out, a) + } + return out +} + +// importPaths returns the import paths to use for the given command line. +func importPaths(args []string) []string { + args = importPathsNoDotExpansion(args) + var out []string + for _, a := range args { + if strings.Contains(a, "...") { + if build.IsLocalImport(a) { + out = append(out, allPackagesInFS(a)...) + } else { + out = append(out, allPackages(a)...) + } + continue + } + out = append(out, a) + } + return out +} + +// matchPattern(pattern)(name) reports whether +// name matches pattern. Pattern is a limited glob +// pattern in which '...' means 'any string' and there +// is no other special syntax. +func matchPattern(pattern string) func(name string) bool { + re := regexp.QuoteMeta(pattern) + re = strings.Replace(re, `\.\.\.`, `.*`, -1) + // Special case: foo/... matches foo too. + if strings.HasSuffix(re, `/.*`) { + re = re[:len(re)-len(`/.*`)] + `(/.*)?` + } + reg := regexp.MustCompile(`^` + re + `$`) + return func(name string) bool { + return reg.MatchString(name) + } +} + +// hasPathPrefix reports whether the path s begins with the +// elements in prefix. +func hasPathPrefix(s, prefix string) bool { + switch { + default: + return false + case len(s) == len(prefix): + return s == prefix + case len(s) > len(prefix): + if prefix != "" && prefix[len(prefix)-1] == '/' { + return strings.HasPrefix(s, prefix) + } + return s[len(prefix)] == '/' && s[:len(prefix)] == prefix + } +} + +// treeCanMatchPattern(pattern)(name) reports whether +// name or children of name can possibly match pattern. +// Pattern is the same limited glob accepted by matchPattern. +func treeCanMatchPattern(pattern string) func(name string) bool { + wildCard := false + if i := strings.Index(pattern, "..."); i >= 0 { + wildCard = true + pattern = pattern[:i] + } + return func(name string) bool { + return len(name) <= len(pattern) && hasPathPrefix(pattern, name) || + wildCard && strings.HasPrefix(name, pattern) + } +} + +// allPackages returns all the packages that can be found +// under the $GOPATH directories and $GOROOT matching pattern. +// The pattern is either "all" (all packages), "std" (standard packages) +// or a path including "...". +func allPackages(pattern string) []string { + pkgs := matchPackages(pattern) + if len(pkgs) == 0 { + fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) + } + return pkgs +} + +func matchPackages(pattern string) []string { + match := func(string) bool { return true } + treeCanMatch := func(string) bool { return true } + if pattern != "all" && pattern != "std" { + match = matchPattern(pattern) + treeCanMatch = treeCanMatchPattern(pattern) + } + + have := map[string]bool{ + "builtin": true, // ignore pseudo-package that exists only for documentation + } + if !buildContext.CgoEnabled { + have["runtime/cgo"] = true // ignore during walk + } + var pkgs []string + + // Commands + cmd := filepath.Join(goroot, "src/cmd") + string(filepath.Separator) + filepath.Walk(cmd, func(path string, fi os.FileInfo, err error) error { + if err != nil || !fi.IsDir() || path == cmd { + return nil + } + name := path[len(cmd):] + if !treeCanMatch(name) { + return filepath.SkipDir + } + // Commands are all in cmd/, not in subdirectories. + if strings.Contains(name, string(filepath.Separator)) { + return filepath.SkipDir + } + + // We use, e.g., cmd/gofmt as the pseudo import path for gofmt. + name = "cmd/" + name + if have[name] { + return nil + } + have[name] = true + if !match(name) { + return nil + } + _, err = buildContext.ImportDir(path, 0) + if err != nil { + if _, noGo := err.(*build.NoGoError); !noGo { + log.Print(err) + } + return nil + } + pkgs = append(pkgs, name) + return nil + }) + + for _, src := range buildContext.SrcDirs() { + if (pattern == "std" || pattern == "cmd") && src != gorootSrc { + continue + } + src = filepath.Clean(src) + string(filepath.Separator) + root := src + if pattern == "cmd" { + root += "cmd" + string(filepath.Separator) + } + filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + if err != nil || !fi.IsDir() || path == src { + return nil + } + + // Avoid .foo, _foo, testdata and vendor directory trees. + _, elem := filepath.Split(path) + if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" || elem == "vendor" { + return filepath.SkipDir + } + + name := filepath.ToSlash(path[len(src):]) + if pattern == "std" && (strings.Contains(name, ".") || name == "cmd") { + // The name "std" is only the standard library. + // If the name is cmd, it's the root of the command tree. + return filepath.SkipDir + } + if !treeCanMatch(name) { + return filepath.SkipDir + } + if have[name] { + return nil + } + have[name] = true + if !match(name) { + return nil + } + _, err = buildContext.ImportDir(path, 0) + if err != nil { + if _, noGo := err.(*build.NoGoError); noGo { + return nil + } + } + pkgs = append(pkgs, name) + return nil + }) + } + return pkgs +} + +// allPackagesInFS is like allPackages but is passed a pattern +// beginning ./ or ../, meaning it should scan the tree rooted +// at the given directory. There are ... in the pattern too. +func allPackagesInFS(pattern string) []string { + pkgs := matchPackagesInFS(pattern) + if len(pkgs) == 0 { + fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) + } + return pkgs +} + +func matchPackagesInFS(pattern string) []string { + // Find directory to begin the scan. + // Could be smarter but this one optimization + // is enough for now, since ... is usually at the + // end of a path. + i := strings.Index(pattern, "...") + dir, _ := path.Split(pattern[:i]) + + // pattern begins with ./ or ../. + // path.Clean will discard the ./ but not the ../. + // We need to preserve the ./ for pattern matching + // and in the returned import paths. + prefix := "" + if strings.HasPrefix(pattern, "./") { + prefix = "./" + } + match := matchPattern(pattern) + + var pkgs []string + filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { + if err != nil || !fi.IsDir() { + return nil + } + if path == dir { + // filepath.Walk starts at dir and recurses. For the recursive case, + // the path is the result of filepath.Join, which calls filepath.Clean. + // The initial case is not Cleaned, though, so we do this explicitly. + // + // This converts a path like "./io/" to "io". Without this step, running + // "cd $GOROOT/src/pkg; go list ./io/..." would incorrectly skip the io + // package, because prepending the prefix "./" to the unclean path would + // result in "././io", and match("././io") returns false. + path = filepath.Clean(path) + } + + // Avoid .foo, _foo, testdata and vendor directory trees, but do not avoid "." or "..". + _, elem := filepath.Split(path) + dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".." + if dot || strings.HasPrefix(elem, "_") || elem == "testdata" || elem == "vendor" { + return filepath.SkipDir + } + + name := prefix + filepath.ToSlash(path) + if !match(name) { + return nil + } + if _, err = build.ImportDir(path, 0); err != nil { + if _, noGo := err.(*build.NoGoError); !noGo { + log.Print(err) + } + return nil + } + pkgs = append(pkgs, name) + return nil + }) + return pkgs +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/alexkohler/nakedret/nakedret.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/alexkohler/nakedret/nakedret.go new file mode 100644 index 000000000..b65177028 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/alexkohler/nakedret/nakedret.go @@ -0,0 +1,213 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "strings" +) + +const ( + pwd = "./" +) + +func init() { + //TODO allow build tags + build.Default.UseAllFiles = true +} + +func usage() { + log.Printf("Usage of %s:\n", os.Args[0]) + log.Printf("\nnakedret [flags] # runs on package in current directory\n") + log.Printf("\nnakedret [flags] [packages]\n") + log.Printf("Flags:\n") + flag.PrintDefaults() +} + +type returnsVisitor struct { + f *token.FileSet + maxLength uint +} + +func main() { + + // Remove log timestamp + log.SetFlags(0) + + maxLength := flag.Uint("l", 5, "maximum number of lines for a naked return function") + flag.Usage = usage + flag.Parse() + + if err := checkNakedReturns(flag.Args(), maxLength); err != nil { + log.Println(err) + } +} + +func checkNakedReturns(args []string, maxLength *uint) error { + + fset := token.NewFileSet() + + files, err := parseInput(args, fset) + if err != nil { + return fmt.Errorf("could not parse input %v", err) + } + + if maxLength == nil { + return errors.New("max length nil") + } + + retVis := &returnsVisitor{ + f: fset, + maxLength: *maxLength, + } + + for _, f := range files { + ast.Walk(retVis, f) + } + + return nil +} + +func parseInput(args []string, fset *token.FileSet) ([]*ast.File, error) { + var directoryList []string + var fileMode bool + files := make([]*ast.File, 0) + + if len(args) == 0 { + directoryList = append(directoryList, pwd) + } else { + for _, arg := range args { + if strings.HasSuffix(arg, "/...") && isDir(arg[:len(arg)-len("/...")]) { + + for _, dirname := range allPackagesInFS(arg) { + directoryList = append(directoryList, dirname) + } + + } else if isDir(arg) { + directoryList = append(directoryList, arg) + + } else if exists(arg) { + if strings.HasSuffix(arg, ".go") { + fileMode = true + f, err := parser.ParseFile(fset, arg, nil, 0) + if err != nil { + return nil, err + } + files = append(files, f) + } else { + return nil, fmt.Errorf("invalid file %v specified", arg) + } + } else { + + //TODO clean this up a bit + imPaths := importPaths([]string{arg}) + for _, importPath := range imPaths { + pkg, err := build.Import(importPath, ".", 0) + if err != nil { + return nil, err + } + var stringFiles []string + stringFiles = append(stringFiles, pkg.GoFiles...) + // files = append(files, pkg.CgoFiles...) + stringFiles = append(stringFiles, pkg.TestGoFiles...) + if pkg.Dir != "." { + for i, f := range stringFiles { + stringFiles[i] = filepath.Join(pkg.Dir, f) + } + } + + fileMode = true + for _, stringFile := range stringFiles { + f, err := parser.ParseFile(fset, stringFile, nil, 0) + if err != nil { + return nil, err + } + files = append(files, f) + } + + } + } + } + } + + // if we're not in file mode, then we need to grab each and every package in each directory + // we can to grab all the files + if !fileMode { + for _, fpath := range directoryList { + pkgs, err := parser.ParseDir(fset, fpath, nil, 0) + if err != nil { + return nil, err + } + + for _, pkg := range pkgs { + for _, f := range pkg.Files { + files = append(files, f) + } + } + } + } + + return files, nil +} + +func isDir(filename string) bool { + fi, err := os.Stat(filename) + return err == nil && fi.IsDir() +} + +func exists(filename string) bool { + _, err := os.Stat(filename) + return err == nil +} + +func (v *returnsVisitor) Visit(node ast.Node) ast.Visitor { + var namedReturns []*ast.Ident + + funcDecl, ok := node.(*ast.FuncDecl) + if !ok { + return v + } + var functionLineLength int + // We've found a function + if funcDecl.Type != nil && funcDecl.Type.Results != nil { + for _, field := range funcDecl.Type.Results.List { + for _, ident := range field.Names { + if ident != nil { + namedReturns = append(namedReturns, ident) + } + } + } + file := v.f.File(funcDecl.Pos()) + functionLineLength = file.Position(funcDecl.End()).Line - file.Position(funcDecl.Pos()).Line + } + + if len(namedReturns) > 0 && funcDecl.Body != nil { + // Scan the body for usage of the named returns + for _, stmt := range funcDecl.Body.List { + + switch s := stmt.(type) { + case *ast.ReturnStmt: + if len(s.Results) == 0 { + file := v.f.File(s.Pos()) + if file != nil && uint(functionLineLength) > v.maxLength { + if funcDecl.Name != nil { + log.Printf("%v:%v %v naked returns on %v line function \n", file.Name(), file.Position(s.Pos()).Line, funcDecl.Name.Name, functionLineLength) + } + } + continue + } + + default: + } + } + } + + return v +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/LICENSE b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/LICENSE new file mode 100644 index 000000000..6a66aea5e --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/asmdecl.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/asmdecl.go new file mode 100644 index 000000000..d543b2ee5 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/asmdecl.go @@ -0,0 +1,682 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Identify mismatches between assembly files and Go func declarations. + +package main + +import ( + "bytes" + "fmt" + "go/ast" + "go/token" + "regexp" + "strconv" + "strings" +) + +// 'kind' is a kind of assembly variable. +// The kinds 1, 2, 4, 8 stand for values of that size. +type asmKind int + +// These special kinds are not valid sizes. +const ( + asmString asmKind = 100 + iota + asmSlice + asmInterface + asmEmptyInterface +) + +// An asmArch describes assembly parameters for an architecture +type asmArch struct { + name string + ptrSize int + intSize int + maxAlign int + bigEndian bool + stack string + lr bool +} + +// An asmFunc describes the expected variables for a function on a given architecture. +type asmFunc struct { + arch *asmArch + size int // size of all arguments + vars map[string]*asmVar + varByOffset map[int]*asmVar +} + +// An asmVar describes a single assembly variable. +type asmVar struct { + name string + kind asmKind + typ string + off int + size int + inner []*asmVar +} + +var ( + asmArch386 = asmArch{"386", 4, 4, 4, false, "SP", false} + asmArchArm = asmArch{"arm", 4, 4, 4, false, "R13", true} + asmArchArm64 = asmArch{"arm64", 8, 8, 8, false, "RSP", true} + asmArchAmd64 = asmArch{"amd64", 8, 8, 8, false, "SP", false} + asmArchAmd64p32 = asmArch{"amd64p32", 4, 4, 8, false, "SP", false} + asmArchMips64 = asmArch{"mips64", 8, 8, 8, true, "R29", true} + asmArchMips64LE = asmArch{"mips64", 8, 8, 8, false, "R29", true} + asmArchPpc64 = asmArch{"ppc64", 8, 8, 8, true, "R1", true} + asmArchPpc64LE = asmArch{"ppc64le", 8, 8, 8, false, "R1", true} + + arches = []*asmArch{ + &asmArch386, + &asmArchArm, + &asmArchArm64, + &asmArchAmd64, + &asmArchAmd64p32, + &asmArchMips64, + &asmArchMips64LE, + &asmArchPpc64, + &asmArchPpc64LE, + } +) + +var ( + re = regexp.MustCompile + asmPlusBuild = re(`//\s+\+build\s+([^\n]+)`) + asmTEXT = re(`\bTEXT\b.*·([^\(]+)\(SB\)(?:\s*,\s*([0-9A-Z|+]+))?(?:\s*,\s*\$(-?[0-9]+)(?:-([0-9]+))?)?`) + asmDATA = re(`\b(DATA|GLOBL)\b`) + asmNamedFP = re(`([a-zA-Z0-9_\xFF-\x{10FFFF}]+)(?:\+([0-9]+))\(FP\)`) + asmUnnamedFP = re(`[^+\-0-9](([0-9]+)\(FP\))`) + asmSP = re(`[^+\-0-9](([0-9]+)\(([A-Z0-9]+)\))`) + asmOpcode = re(`^\s*(?:[A-Z0-9a-z_]+:)?\s*([A-Z]+)\s*([^,]*)(?:,\s*(.*))?`) + ppc64Suff = re(`([BHWD])(ZU|Z|U|BR)?$`) +) + +func asmCheck(pkg *Package) { + if !vet("asmdecl") { + return + } + + // No work if no assembly files. + if !pkg.hasFileWithSuffix(".s") { + return + } + + // Gather declarations. knownFunc[name][arch] is func description. + knownFunc := make(map[string]map[string]*asmFunc) + + for _, f := range pkg.files { + if f.file != nil { + for _, decl := range f.file.Decls { + if decl, ok := decl.(*ast.FuncDecl); ok && decl.Body == nil { + knownFunc[decl.Name.Name] = f.asmParseDecl(decl) + } + } + } + } + +Files: + for _, f := range pkg.files { + if !strings.HasSuffix(f.name, ".s") { + continue + } + Println("Checking file", f.name) + + // Determine architecture from file name if possible. + var arch string + var archDef *asmArch + for _, a := range arches { + if strings.HasSuffix(f.name, "_"+a.name+".s") { + arch = a.name + archDef = a + break + } + } + + lines := strings.SplitAfter(string(f.content), "\n") + var ( + fn *asmFunc + fnName string + localSize, argSize int + wroteSP bool + haveRetArg bool + retLine []int + ) + + flushRet := func() { + if fn != nil && fn.vars["ret"] != nil && !haveRetArg && len(retLine) > 0 { + v := fn.vars["ret"] + for _, line := range retLine { + f.Badf(token.NoPos, "%s:%d: [%s] %s: RET without writing to %d-byte ret+%d(FP)", f.name, line, arch, fnName, v.size, v.off) + } + } + retLine = nil + } + for lineno, line := range lines { + lineno++ + + badf := func(format string, args ...interface{}) { + f.Badf(token.NoPos, "%s:%d: [%s] %s: %s", f.name, lineno, arch, fnName, fmt.Sprintf(format, args...)) + } + + if arch == "" { + // Determine architecture from +build line if possible. + if m := asmPlusBuild.FindStringSubmatch(line); m != nil { + Fields: + for _, fld := range strings.Fields(m[1]) { + for _, a := range arches { + if a.name == fld { + arch = a.name + archDef = a + break Fields + } + } + } + } + } + + if m := asmTEXT.FindStringSubmatch(line); m != nil { + flushRet() + if arch == "" { + f.Warnf(token.NoPos, "%s: cannot determine architecture for assembly file", f.name) + continue Files + } + fnName = m[1] + fn = knownFunc[m[1]][arch] + if fn != nil { + size, _ := strconv.Atoi(m[4]) + if size != fn.size && (m[2] != "7" && !strings.Contains(m[2], "NOSPLIT") || size != 0) { + badf("wrong argument size %d; expected $...-%d", size, fn.size) + } + } + localSize, _ = strconv.Atoi(m[3]) + localSize += archDef.intSize + if archDef.lr { + // Account for caller's saved LR + localSize += archDef.intSize + } + argSize, _ = strconv.Atoi(m[4]) + if fn == nil && !strings.Contains(fnName, "<>") { + badf("function %s missing Go declaration", fnName) + } + wroteSP = false + haveRetArg = false + continue + } else if strings.Contains(line, "TEXT") && strings.Contains(line, "SB") { + // function, but not visible from Go (didn't match asmTEXT), so stop checking + flushRet() + fn = nil + fnName = "" + continue + } + + if strings.Contains(line, "RET") { + retLine = append(retLine, lineno) + } + + if fnName == "" { + continue + } + + if asmDATA.FindStringSubmatch(line) != nil { + fn = nil + } + + if archDef == nil { + continue + } + + if strings.Contains(line, ", "+archDef.stack) || strings.Contains(line, ",\t"+archDef.stack) { + wroteSP = true + continue + } + + for _, m := range asmSP.FindAllStringSubmatch(line, -1) { + if m[3] != archDef.stack || wroteSP { + continue + } + off := 0 + if m[1] != "" { + off, _ = strconv.Atoi(m[2]) + } + if off >= localSize { + if fn != nil { + v := fn.varByOffset[off-localSize] + if v != nil { + badf("%s should be %s+%d(FP)", m[1], v.name, off-localSize) + continue + } + } + if off >= localSize+argSize { + badf("use of %s points beyond argument frame", m[1]) + continue + } + badf("use of %s to access argument frame", m[1]) + } + } + + if fn == nil { + continue + } + + for _, m := range asmUnnamedFP.FindAllStringSubmatch(line, -1) { + off, _ := strconv.Atoi(m[2]) + v := fn.varByOffset[off] + if v != nil { + badf("use of unnamed argument %s; offset %d is %s+%d(FP)", m[1], off, v.name, v.off) + } else { + badf("use of unnamed argument %s", m[1]) + } + } + + for _, m := range asmNamedFP.FindAllStringSubmatch(line, -1) { + name := m[1] + off := 0 + if m[2] != "" { + off, _ = strconv.Atoi(m[2]) + } + if name == "ret" || strings.HasPrefix(name, "ret_") { + haveRetArg = true + } + v := fn.vars[name] + if v == nil { + // Allow argframe+0(FP). + if name == "argframe" && off == 0 { + continue + } + v = fn.varByOffset[off] + if v != nil { + badf("unknown variable %s; offset %d is %s+%d(FP)", name, off, v.name, v.off) + } else { + badf("unknown variable %s", name) + } + continue + } + asmCheckVar(badf, fn, line, m[0], off, v) + } + } + flushRet() + } +} + +// asmParseDecl parses a function decl for expected assembly variables. +func (f *File) asmParseDecl(decl *ast.FuncDecl) map[string]*asmFunc { + var ( + arch *asmArch + fn *asmFunc + offset int + failed bool + ) + + addVar := func(outer string, v asmVar) { + if vo := fn.vars[outer]; vo != nil { + vo.inner = append(vo.inner, &v) + } + fn.vars[v.name] = &v + for i := 0; i < v.size; i++ { + fn.varByOffset[v.off+i] = &v + } + } + + addParams := func(list []*ast.Field) { + for i, fld := range list { + // Determine alignment, size, and kind of type in declaration. + var align, size int + var kind asmKind + names := fld.Names + typ := f.gofmt(fld.Type) + switch t := fld.Type.(type) { + default: + switch typ { + default: + f.Warnf(fld.Type.Pos(), "unknown assembly argument type %s", typ) + failed = true + return + case "int8", "uint8", "byte", "bool": + size = 1 + case "int16", "uint16": + size = 2 + case "int32", "uint32", "float32": + size = 4 + case "int64", "uint64", "float64": + align = arch.maxAlign + size = 8 + case "int", "uint": + size = arch.intSize + case "uintptr", "iword", "Word", "Errno", "unsafe.Pointer": + size = arch.ptrSize + case "string", "ErrorString": + size = arch.ptrSize * 2 + align = arch.ptrSize + kind = asmString + } + case *ast.ChanType, *ast.FuncType, *ast.MapType, *ast.StarExpr: + size = arch.ptrSize + case *ast.InterfaceType: + align = arch.ptrSize + size = 2 * arch.ptrSize + if len(t.Methods.List) > 0 { + kind = asmInterface + } else { + kind = asmEmptyInterface + } + case *ast.ArrayType: + if t.Len == nil { + size = arch.ptrSize + 2*arch.intSize + align = arch.ptrSize + kind = asmSlice + break + } + f.Warnf(fld.Type.Pos(), "unsupported assembly argument type %s", typ) + failed = true + case *ast.StructType: + f.Warnf(fld.Type.Pos(), "unsupported assembly argument type %s", typ) + failed = true + } + if align == 0 { + align = size + } + if kind == 0 { + kind = asmKind(size) + } + offset += -offset & (align - 1) + + // Create variable for each name being declared with this type. + if len(names) == 0 { + name := "unnamed" + if decl.Type.Results != nil && len(decl.Type.Results.List) > 0 && &list[0] == &decl.Type.Results.List[0] && i == 0 { + // Assume assembly will refer to single unnamed result as r. + name = "ret" + } + names = []*ast.Ident{{Name: name}} + } + for _, id := range names { + name := id.Name + addVar("", asmVar{ + name: name, + kind: kind, + typ: typ, + off: offset, + size: size, + }) + switch kind { + case 8: + if arch.ptrSize == 4 { + w1, w2 := "lo", "hi" + if arch.bigEndian { + w1, w2 = w2, w1 + } + addVar(name, asmVar{ + name: name + "_" + w1, + kind: 4, + typ: "half " + typ, + off: offset, + size: 4, + }) + addVar(name, asmVar{ + name: name + "_" + w2, + kind: 4, + typ: "half " + typ, + off: offset + 4, + size: 4, + }) + } + + case asmEmptyInterface: + addVar(name, asmVar{ + name: name + "_type", + kind: asmKind(arch.ptrSize), + typ: "interface type", + off: offset, + size: arch.ptrSize, + }) + addVar(name, asmVar{ + name: name + "_data", + kind: asmKind(arch.ptrSize), + typ: "interface data", + off: offset + arch.ptrSize, + size: arch.ptrSize, + }) + + case asmInterface: + addVar(name, asmVar{ + name: name + "_itable", + kind: asmKind(arch.ptrSize), + typ: "interface itable", + off: offset, + size: arch.ptrSize, + }) + addVar(name, asmVar{ + name: name + "_data", + kind: asmKind(arch.ptrSize), + typ: "interface data", + off: offset + arch.ptrSize, + size: arch.ptrSize, + }) + + case asmSlice: + addVar(name, asmVar{ + name: name + "_base", + kind: asmKind(arch.ptrSize), + typ: "slice base", + off: offset, + size: arch.ptrSize, + }) + addVar(name, asmVar{ + name: name + "_len", + kind: asmKind(arch.intSize), + typ: "slice len", + off: offset + arch.ptrSize, + size: arch.intSize, + }) + addVar(name, asmVar{ + name: name + "_cap", + kind: asmKind(arch.intSize), + typ: "slice cap", + off: offset + arch.ptrSize + arch.intSize, + size: arch.intSize, + }) + + case asmString: + addVar(name, asmVar{ + name: name + "_base", + kind: asmKind(arch.ptrSize), + typ: "string base", + off: offset, + size: arch.ptrSize, + }) + addVar(name, asmVar{ + name: name + "_len", + kind: asmKind(arch.intSize), + typ: "string len", + off: offset + arch.ptrSize, + size: arch.intSize, + }) + } + offset += size + } + } + } + + m := make(map[string]*asmFunc) + for _, arch = range arches { + fn = &asmFunc{ + arch: arch, + vars: make(map[string]*asmVar), + varByOffset: make(map[int]*asmVar), + } + offset = 0 + addParams(decl.Type.Params.List) + if decl.Type.Results != nil && len(decl.Type.Results.List) > 0 { + offset += -offset & (arch.maxAlign - 1) + addParams(decl.Type.Results.List) + } + fn.size = offset + m[arch.name] = fn + } + + if failed { + return nil + } + return m +} + +// asmCheckVar checks a single variable reference. +func asmCheckVar(badf func(string, ...interface{}), fn *asmFunc, line, expr string, off int, v *asmVar) { + m := asmOpcode.FindStringSubmatch(line) + if m == nil { + if !strings.HasPrefix(strings.TrimSpace(line), "//") { + badf("cannot find assembly opcode") + } + return + } + + // Determine operand sizes from instruction. + // Typically the suffix suffices, but there are exceptions. + var src, dst, kind asmKind + op := m[1] + switch fn.arch.name + "." + op { + case "386.FMOVLP": + src, dst = 8, 4 + case "arm.MOVD": + src = 8 + case "arm.MOVW": + src = 4 + case "arm.MOVH", "arm.MOVHU": + src = 2 + case "arm.MOVB", "arm.MOVBU": + src = 1 + // LEA* opcodes don't really read the second arg. + // They just take the address of it. + case "386.LEAL": + dst = 4 + case "amd64.LEAQ": + dst = 8 + case "amd64p32.LEAL": + dst = 4 + default: + switch fn.arch.name { + case "386", "amd64": + if strings.HasPrefix(op, "F") && (strings.HasSuffix(op, "D") || strings.HasSuffix(op, "DP")) { + // FMOVDP, FXCHD, etc + src = 8 + break + } + if strings.HasPrefix(op, "P") && strings.HasSuffix(op, "RD") { + // PINSRD, PEXTRD, etc + src = 4 + break + } + if strings.HasPrefix(op, "F") && (strings.HasSuffix(op, "F") || strings.HasSuffix(op, "FP")) { + // FMOVFP, FXCHF, etc + src = 4 + break + } + if strings.HasSuffix(op, "SD") { + // MOVSD, SQRTSD, etc + src = 8 + break + } + if strings.HasSuffix(op, "SS") { + // MOVSS, SQRTSS, etc + src = 4 + break + } + if strings.HasPrefix(op, "SET") { + // SETEQ, etc + src = 1 + break + } + switch op[len(op)-1] { + case 'B': + src = 1 + case 'W': + src = 2 + case 'L': + src = 4 + case 'D', 'Q': + src = 8 + } + case "ppc64", "ppc64le": + // Strip standard suffixes to reveal size letter. + m := ppc64Suff.FindStringSubmatch(op) + if m != nil { + switch m[1][0] { + case 'B': + src = 1 + case 'H': + src = 2 + case 'W': + src = 4 + case 'D': + src = 8 + } + } + case "mips64", "mips64le": + switch op { + case "MOVB", "MOVBU": + src = 1 + case "MOVH", "MOVHU": + src = 2 + case "MOVW", "MOVWU", "MOVF": + src = 4 + case "MOVV", "MOVD": + src = 8 + } + } + } + if dst == 0 { + dst = src + } + + // Determine whether the match we're holding + // is the first or second argument. + if strings.Index(line, expr) > strings.Index(line, ",") { + kind = dst + } else { + kind = src + } + + vk := v.kind + vt := v.typ + switch vk { + case asmInterface, asmEmptyInterface, asmString, asmSlice: + // allow reference to first word (pointer) + vk = v.inner[0].kind + vt = v.inner[0].typ + } + + if off != v.off { + var inner bytes.Buffer + for i, vi := range v.inner { + if len(v.inner) > 1 { + fmt.Fprintf(&inner, ",") + } + fmt.Fprintf(&inner, " ") + if i == len(v.inner)-1 { + fmt.Fprintf(&inner, "or ") + } + fmt.Fprintf(&inner, "%s+%d(FP)", vi.name, vi.off) + } + badf("invalid offset %s; expected %s+%d(FP)%s", expr, v.name, v.off, inner.String()) + return + } + if kind != 0 && kind != vk { + var inner bytes.Buffer + if len(v.inner) > 0 { + fmt.Fprintf(&inner, " containing") + for i, vi := range v.inner { + if i > 0 && len(v.inner) > 2 { + fmt.Fprintf(&inner, ",") + } + fmt.Fprintf(&inner, " ") + if i > 0 && i == len(v.inner)-1 { + fmt.Fprintf(&inner, "and ") + } + fmt.Fprintf(&inner, "%s+%d(FP)", vi.name, vi.off) + } + } + badf("invalid %s of %s; %s is %d-byte value%s", op, expr, vt, vk, inner.String()) + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/assign.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/assign.go new file mode 100644 index 000000000..54c1ae1fd --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/assign.go @@ -0,0 +1,49 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +This file contains the code to check for useless assignments. +*/ + +package main + +import ( + "go/ast" + "go/token" + "reflect" +) + +func init() { + register("assign", + "check for useless assignments", + checkAssignStmt, + assignStmt) +} + +// TODO: should also check for assignments to struct fields inside methods +// that are on T instead of *T. + +// checkAssignStmt checks for assignments of the form " = ". +// These are almost always useless, and even when they aren't they are usually a mistake. +func checkAssignStmt(f *File, node ast.Node) { + stmt := node.(*ast.AssignStmt) + if stmt.Tok != token.ASSIGN { + return // ignore := + } + if len(stmt.Lhs) != len(stmt.Rhs) { + // If LHS and RHS have different cardinality, they can't be the same. + return + } + for i, lhs := range stmt.Lhs { + rhs := stmt.Rhs[i] + if reflect.TypeOf(lhs) != reflect.TypeOf(rhs) { + continue // short-circuit the heavy-weight gofmt check + } + le := f.gofmt(lhs) + re := f.gofmt(rhs) + if le == re { + f.Badf(stmt.Pos(), "self-assignment of %s to %s", re, le) + } + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/atomic.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/atomic.go new file mode 100644 index 000000000..b2ca2d80f --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/atomic.go @@ -0,0 +1,69 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" + "go/token" +) + +func init() { + register("atomic", + "check for common mistaken usages of the sync/atomic package", + checkAtomicAssignment, + assignStmt) +} + +// checkAtomicAssignment walks the assignment statement checking for common +// mistaken usage of atomic package, such as: x = atomic.AddUint64(&x, 1) +func checkAtomicAssignment(f *File, node ast.Node) { + n := node.(*ast.AssignStmt) + if len(n.Lhs) != len(n.Rhs) { + return + } + if len(n.Lhs) == 1 && n.Tok == token.DEFINE { + return + } + + for i, right := range n.Rhs { + call, ok := right.(*ast.CallExpr) + if !ok { + continue + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + continue + } + pkg, ok := sel.X.(*ast.Ident) + if !ok || pkg.Name != "atomic" { + continue + } + + switch sel.Sel.Name { + case "AddInt32", "AddInt64", "AddUint32", "AddUint64", "AddUintptr": + f.checkAtomicAddAssignment(n.Lhs[i], call) + } + } +} + +// checkAtomicAddAssignment walks the atomic.Add* method calls checking for assigning the return value +// to the same variable being used in the operation +func (f *File) checkAtomicAddAssignment(left ast.Expr, call *ast.CallExpr) { + if len(call.Args) != 2 { + return + } + arg := call.Args[0] + broken := false + + if uarg, ok := arg.(*ast.UnaryExpr); ok && uarg.Op == token.AND { + broken = f.gofmt(left) == f.gofmt(uarg.X) + } else if star, ok := left.(*ast.StarExpr); ok { + broken = f.gofmt(star.X) == f.gofmt(arg) + } + + if broken { + f.Bad(left.Pos(), "direct assignment to atomic value") + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/bool.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/bool.go new file mode 100644 index 000000000..07c2a93df --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/bool.go @@ -0,0 +1,186 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains boolean condition tests. + +package main + +import ( + "go/ast" + "go/token" +) + +func init() { + register("bool", + "check for mistakes involving boolean operators", + checkBool, + binaryExpr) +} + +func checkBool(f *File, n ast.Node) { + e := n.(*ast.BinaryExpr) + + var op boolOp + switch e.Op { + case token.LOR: + op = or + case token.LAND: + op = and + default: + return + } + + comm := op.commutativeSets(e) + for _, exprs := range comm { + op.checkRedundant(f, exprs) + op.checkSuspect(f, exprs) + } +} + +type boolOp struct { + name string + tok token.Token // token corresponding to this operator + badEq token.Token // token corresponding to the equality test that should not be used with this operator +} + +var ( + or = boolOp{"or", token.LOR, token.NEQ} + and = boolOp{"and", token.LAND, token.EQL} +) + +// commutativeSets returns all side effect free sets of +// expressions in e that are connected by op. +// For example, given 'a || b || f() || c || d' with the or op, +// commutativeSets returns {{b, a}, {d, c}}. +func (op boolOp) commutativeSets(e *ast.BinaryExpr) [][]ast.Expr { + exprs := op.split(e) + + // Partition the slice of expressions into commutative sets. + i := 0 + var sets [][]ast.Expr + for j := 0; j <= len(exprs); j++ { + if j == len(exprs) || hasSideEffects(exprs[j]) { + if i < j { + sets = append(sets, exprs[i:j]) + } + i = j + 1 + } + } + + return sets +} + +// checkRedundant checks for expressions of the form +// e && e +// e || e +// Exprs must contain only side effect free expressions. +func (op boolOp) checkRedundant(f *File, exprs []ast.Expr) { + seen := make(map[string]bool) + for _, e := range exprs { + efmt := f.gofmt(e) + if seen[efmt] { + f.Badf(e.Pos(), "redundant %s: %s %s %s", op.name, efmt, op.tok, efmt) + } else { + seen[efmt] = true + } + } +} + +// checkSuspect checks for expressions of the form +// x != c1 || x != c2 +// x == c1 && x == c2 +// where c1 and c2 are constant expressions. +// If c1 and c2 are the same then it's redundant; +// if c1 and c2 are different then it's always true or always false. +// Exprs must contain only side effect free expressions. +func (op boolOp) checkSuspect(f *File, exprs []ast.Expr) { + // seen maps from expressions 'x' to equality expressions 'x != c'. + seen := make(map[string]string) + + for _, e := range exprs { + bin, ok := e.(*ast.BinaryExpr) + if !ok || bin.Op != op.badEq { + continue + } + + // In order to avoid false positives, restrict to cases + // in which one of the operands is constant. We're then + // interested in the other operand. + // In the rare case in which both operands are constant + // (e.g. runtime.GOOS and "windows"), we'll only catch + // mistakes if the LHS is repeated, which is how most + // code is written. + var x ast.Expr + switch { + case f.pkg.types[bin.Y].Value != nil: + x = bin.X + case f.pkg.types[bin.X].Value != nil: + x = bin.Y + default: + continue + } + + // e is of the form 'x != c' or 'x == c'. + xfmt := f.gofmt(x) + efmt := f.gofmt(e) + if prev, found := seen[xfmt]; found { + // checkRedundant handles the case in which efmt == prev. + if efmt != prev { + f.Badf(e.Pos(), "suspect %s: %s %s %s", op.name, efmt, op.tok, prev) + } + } else { + seen[xfmt] = efmt + } + } +} + +// hasSideEffects reports whether evaluation of e has side effects. +func hasSideEffects(e ast.Expr) bool { + safe := true + ast.Inspect(e, func(node ast.Node) bool { + switch n := node.(type) { + // Using CallExpr here will catch conversions + // as well as function and method invocations. + // We'll live with the false negatives for now. + case *ast.CallExpr: + safe = false + return false + case *ast.UnaryExpr: + if n.Op == token.ARROW { + safe = false + return false + } + } + return true + }) + return !safe +} + +// split returns a slice of all subexpressions in e that are connected by op. +// For example, given 'a || (b || c) || d' with the or op, +// split returns []{d, c, b, a}. +func (op boolOp) split(e ast.Expr) (exprs []ast.Expr) { + for { + e = unparen(e) + if b, ok := e.(*ast.BinaryExpr); ok && b.Op == op.tok { + exprs = append(exprs, op.split(b.Y)...) + e = b.X + } else { + exprs = append(exprs, e) + break + } + } + return +} + +// unparen returns e with any enclosing parentheses stripped. +func unparen(e ast.Expr) ast.Expr { + for { + p, ok := e.(*ast.ParenExpr) + if !ok { + return e + } + e = p.X + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/buildtag.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/buildtag.go new file mode 100644 index 000000000..ccf764ef8 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/buildtag.go @@ -0,0 +1,91 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "fmt" + "os" + "strings" + "unicode" +) + +var ( + nl = []byte("\n") + slashSlash = []byte("//") + plusBuild = []byte("+build") +) + +// checkBuildTag checks that build tags are in the correct location and well-formed. +func checkBuildTag(name string, data []byte) { + if !vet("buildtags") { + return + } + lines := bytes.SplitAfter(data, nl) + + // Determine cutpoint where +build comments are no longer valid. + // They are valid in leading // comments in the file followed by + // a blank line. + var cutoff int + for i, line := range lines { + line = bytes.TrimSpace(line) + if len(line) == 0 { + cutoff = i + continue + } + if bytes.HasPrefix(line, slashSlash) { + continue + } + break + } + + for i, line := range lines { + line = bytes.TrimSpace(line) + if !bytes.HasPrefix(line, slashSlash) { + continue + } + text := bytes.TrimSpace(line[2:]) + if bytes.HasPrefix(text, plusBuild) { + fields := bytes.Fields(text) + if !bytes.Equal(fields[0], plusBuild) { + // Comment is something like +buildasdf not +build. + fmt.Fprintf(os.Stderr, "%s:%d: possible malformed +build comment\n", name, i+1) + continue + } + if i >= cutoff { + fmt.Fprintf(os.Stderr, "%s:%d: +build comment must appear before package clause and be followed by a blank line\n", name, i+1) + setExit(1) + continue + } + // Check arguments. + Args: + for _, arg := range fields[1:] { + for _, elem := range strings.Split(string(arg), ",") { + if strings.HasPrefix(elem, "!!") { + fmt.Fprintf(os.Stderr, "%s:%d: invalid double negative in build constraint: %s\n", name, i+1, arg) + setExit(1) + break Args + } + if strings.HasPrefix(elem, "!") { + elem = elem[1:] + } + for _, c := range elem { + if !unicode.IsLetter(c) && !unicode.IsDigit(c) && c != '_' && c != '.' { + fmt.Fprintf(os.Stderr, "%s:%d: invalid non-alphanumeric build constraint: %s\n", name, i+1, arg) + setExit(1) + break Args + } + } + } + } + continue + } + // Comment with +build but not at beginning. + if bytes.Contains(line, plusBuild) && i < cutoff { + fmt.Fprintf(os.Stderr, "%s:%d: possible malformed +build comment\n", name, i+1) + continue + } + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/cgo.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/cgo.go new file mode 100644 index 000000000..b896862c8 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/cgo.go @@ -0,0 +1,130 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Check for invalid cgo pointer passing. +// This looks for code that uses cgo to call C code passing values +// whose types are almost always invalid according to the cgo pointer +// sharing rules. +// Specifically, it warns about attempts to pass a Go chan, map, func, +// or slice to C, either directly, or via a pointer, array, or struct. + +package main + +import ( + "go/ast" + "go/token" + "go/types" +) + +func init() { + register("cgocall", + "check for types that may not be passed to cgo calls", + checkCgoCall, + callExpr) +} + +func checkCgoCall(f *File, node ast.Node) { + x := node.(*ast.CallExpr) + + // We are only looking for calls to functions imported from + // the "C" package. + sel, ok := x.Fun.(*ast.SelectorExpr) + if !ok { + return + } + id, ok := sel.X.(*ast.Ident) + if !ok || id.Name != "C" { + return + } + + for _, arg := range x.Args { + if !typeOKForCgoCall(cgoBaseType(f, arg)) { + f.Badf(arg.Pos(), "possibly passing Go type with embedded pointer to C") + } + + // Check for passing the address of a bad type. + if conv, ok := arg.(*ast.CallExpr); ok && len(conv.Args) == 1 && f.hasBasicType(conv.Fun, types.UnsafePointer) { + arg = conv.Args[0] + } + if u, ok := arg.(*ast.UnaryExpr); ok && u.Op == token.AND { + if !typeOKForCgoCall(cgoBaseType(f, u.X)) { + f.Badf(arg.Pos(), "possibly passing Go type with embedded pointer to C") + } + } + } +} + +// cgoBaseType tries to look through type conversions involving +// unsafe.Pointer to find the real type. It converts: +// unsafe.Pointer(x) => x +// *(*unsafe.Pointer)(unsafe.Pointer(&x)) => x +func cgoBaseType(f *File, arg ast.Expr) types.Type { + switch arg := arg.(type) { + case *ast.CallExpr: + if len(arg.Args) == 1 && f.hasBasicType(arg.Fun, types.UnsafePointer) { + return cgoBaseType(f, arg.Args[0]) + } + case *ast.StarExpr: + call, ok := arg.X.(*ast.CallExpr) + if !ok || len(call.Args) != 1 { + break + } + // Here arg is *f(v). + t := f.pkg.types[call.Fun].Type + if t == nil { + break + } + ptr, ok := t.Underlying().(*types.Pointer) + if !ok { + break + } + // Here arg is *(*p)(v) + elem, ok := ptr.Elem().Underlying().(*types.Basic) + if !ok || elem.Kind() != types.UnsafePointer { + break + } + // Here arg is *(*unsafe.Pointer)(v) + call, ok = call.Args[0].(*ast.CallExpr) + if !ok || len(call.Args) != 1 { + break + } + // Here arg is *(*unsafe.Pointer)(f(v)) + if !f.hasBasicType(call.Fun, types.UnsafePointer) { + break + } + // Here arg is *(*unsafe.Pointer)(unsafe.Pointer(v)) + u, ok := call.Args[0].(*ast.UnaryExpr) + if !ok || u.Op != token.AND { + break + } + // Here arg is *(*unsafe.Pointer)(unsafe.Pointer(&v)) + return cgoBaseType(f, u.X) + } + + return f.pkg.types[arg].Type +} + +// typeOKForCgoCall returns true if the type of arg is OK to pass to a +// C function using cgo. This is not true for Go types with embedded +// pointers. +func typeOKForCgoCall(t types.Type) bool { + if t == nil { + return true + } + switch t := t.Underlying().(type) { + case *types.Chan, *types.Map, *types.Signature, *types.Slice: + return false + case *types.Pointer: + return typeOKForCgoCall(t.Elem()) + case *types.Array: + return typeOKForCgoCall(t.Elem()) + case *types.Struct: + for i := 0; i < t.NumFields(); i++ { + if !typeOKForCgoCall(t.Field(i).Type()) { + return false + } + } + } + return true +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/composite.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/composite.go new file mode 100644 index 000000000..8594c4897 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/composite.go @@ -0,0 +1,82 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the test for unkeyed struct literals. + +package main + +import ( + "github.com/dnephin/govet/internal/whitelist" + "flag" + "go/ast" + "go/types" + "strings" +) + +var compositeWhiteList = flag.Bool("compositewhitelist", true, "use composite white list; for testing only") + +func init() { + register("composites", + "check that composite literals used field-keyed elements", + checkUnkeyedLiteral, + compositeLit) +} + +// checkUnkeyedLiteral checks if a composite literal is a struct literal with +// unkeyed fields. +func checkUnkeyedLiteral(f *File, node ast.Node) { + cl := node.(*ast.CompositeLit) + + typ := f.pkg.types[cl].Type + if typ == nil { + // cannot determine composite literals' type, skip it + return + } + typeName := typ.String() + if *compositeWhiteList && whitelist.UnkeyedLiteral[typeName] { + // skip whitelisted types + return + } + if _, ok := typ.Underlying().(*types.Struct); !ok { + // skip non-struct composite literals + return + } + if isLocalType(f, typeName) { + // allow unkeyed locally defined composite literal + return + } + + // check if the CompositeLit contains an unkeyed field + allKeyValue := true + for _, e := range cl.Elts { + if _, ok := e.(*ast.KeyValueExpr); !ok { + allKeyValue = false + break + } + } + if allKeyValue { + // all the composite literal fields are keyed + return + } + + f.Badf(cl.Pos(), "%s composite literal uses unkeyed fields", typeName) +} + +func isLocalType(f *File, typeName string) bool { + if strings.HasPrefix(typeName, "struct{") { + // struct literals are local types + return true + } + + pkgname := f.pkg.path + if strings.HasPrefix(typeName, pkgname+".") { + return true + } + + // treat types as local inside test packages with _test name suffix + if strings.HasSuffix(pkgname, "_test") { + pkgname = pkgname[:len(pkgname)-len("_test")] + } + return strings.HasPrefix(typeName, pkgname+".") +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/copylock.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/copylock.go new file mode 100644 index 000000000..65337688b --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/copylock.go @@ -0,0 +1,239 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the code to check that locks are not passed by value. + +package main + +import ( + "bytes" + "fmt" + "go/ast" + "go/token" + "go/types" +) + +func init() { + register("copylocks", + "check that locks are not passed by value", + checkCopyLocks, + funcDecl, rangeStmt, funcLit, callExpr, assignStmt, genDecl, compositeLit, returnStmt) +} + +// checkCopyLocks checks whether node might +// inadvertently copy a lock. +func checkCopyLocks(f *File, node ast.Node) { + switch node := node.(type) { + case *ast.RangeStmt: + checkCopyLocksRange(f, node) + case *ast.FuncDecl: + checkCopyLocksFunc(f, node.Name.Name, node.Recv, node.Type) + case *ast.FuncLit: + checkCopyLocksFunc(f, "func", nil, node.Type) + case *ast.CallExpr: + checkCopyLocksCallExpr(f, node) + case *ast.AssignStmt: + checkCopyLocksAssign(f, node) + case *ast.GenDecl: + checkCopyLocksGenDecl(f, node) + case *ast.CompositeLit: + checkCopyLocksCompositeLit(f, node) + case *ast.ReturnStmt: + checkCopyLocksReturnStmt(f, node) + } +} + +// checkCopyLocksAssign checks whether an assignment +// copies a lock. +func checkCopyLocksAssign(f *File, as *ast.AssignStmt) { + for i, x := range as.Rhs { + if path := lockPathRhs(f, x); path != nil { + f.Badf(x.Pos(), "assignment copies lock value to %v: %v", f.gofmt(as.Lhs[i]), path) + } + } +} + +// checkCopyLocksGenDecl checks whether lock is copied +// in variable declaration. +func checkCopyLocksGenDecl(f *File, gd *ast.GenDecl) { + if gd.Tok != token.VAR { + return + } + for _, spec := range gd.Specs { + valueSpec := spec.(*ast.ValueSpec) + for i, x := range valueSpec.Values { + if path := lockPathRhs(f, x); path != nil { + f.Badf(x.Pos(), "variable declaration copies lock value to %v: %v", valueSpec.Names[i].Name, path) + } + } + } +} + +// checkCopyLocksCompositeLit detects lock copy inside a composite literal +func checkCopyLocksCompositeLit(f *File, cl *ast.CompositeLit) { + for _, x := range cl.Elts { + if node, ok := x.(*ast.KeyValueExpr); ok { + x = node.Value + } + if path := lockPathRhs(f, x); path != nil { + f.Badf(x.Pos(), "literal copies lock value from %v: %v", f.gofmt(x), path) + } + } +} + +// checkCopyLocksReturnStmt detects lock copy in return statement +func checkCopyLocksReturnStmt(f *File, rs *ast.ReturnStmt) { + for _, x := range rs.Results { + if path := lockPathRhs(f, x); path != nil { + f.Badf(x.Pos(), "return copies lock value: %v", path) + } + } +} + +// checkCopyLocksCallExpr detects lock copy in the arguments to a function call +func checkCopyLocksCallExpr(f *File, ce *ast.CallExpr) { + if id, ok := ce.Fun.(*ast.Ident); ok && id.Name == "new" && f.pkg.types[id].IsBuiltin() { + // Skip 'new(Type)' for built-in 'new' + return + } + for _, x := range ce.Args { + if path := lockPathRhs(f, x); path != nil { + f.Badf(x.Pos(), "function call copies lock value: %v", path) + } + } +} + +// checkCopyLocksFunc checks whether a function might +// inadvertently copy a lock, by checking whether +// its receiver, parameters, or return values +// are locks. +func checkCopyLocksFunc(f *File, name string, recv *ast.FieldList, typ *ast.FuncType) { + if recv != nil && len(recv.List) > 0 { + expr := recv.List[0].Type + if path := lockPath(f.pkg.typesPkg, f.pkg.types[expr].Type); path != nil { + f.Badf(expr.Pos(), "%s passes lock by value: %v", name, path) + } + } + + if typ.Params != nil { + for _, field := range typ.Params.List { + expr := field.Type + if path := lockPath(f.pkg.typesPkg, f.pkg.types[expr].Type); path != nil { + f.Badf(expr.Pos(), "%s passes lock by value: %v", name, path) + } + } + } + + // Don't check typ.Results. If T has a Lock field it's OK to write + // return T{} + // because that is returning the zero value. Leave result checking + // to the return statement. +} + +// checkCopyLocksRange checks whether a range statement +// might inadvertently copy a lock by checking whether +// any of the range variables are locks. +func checkCopyLocksRange(f *File, r *ast.RangeStmt) { + checkCopyLocksRangeVar(f, r.Tok, r.Key) + checkCopyLocksRangeVar(f, r.Tok, r.Value) +} + +func checkCopyLocksRangeVar(f *File, rtok token.Token, e ast.Expr) { + if e == nil { + return + } + id, isId := e.(*ast.Ident) + if isId && id.Name == "_" { + return + } + + var typ types.Type + if rtok == token.DEFINE { + if !isId { + return + } + obj := f.pkg.defs[id] + if obj == nil { + return + } + typ = obj.Type() + } else { + typ = f.pkg.types[e].Type + } + + if typ == nil { + return + } + if path := lockPath(f.pkg.typesPkg, typ); path != nil { + f.Badf(e.Pos(), "range var %s copies lock: %v", f.gofmt(e), path) + } +} + +type typePath []types.Type + +// String pretty-prints a typePath. +func (path typePath) String() string { + n := len(path) + var buf bytes.Buffer + for i := range path { + if i > 0 { + fmt.Fprint(&buf, " contains ") + } + // The human-readable path is in reverse order, outermost to innermost. + fmt.Fprint(&buf, path[n-i-1].String()) + } + return buf.String() +} + +func lockPathRhs(f *File, x ast.Expr) typePath { + if _, ok := x.(*ast.CompositeLit); ok { + return nil + } + if _, ok := x.(*ast.CallExpr); ok { + // A call may return a zero value. + return nil + } + if star, ok := x.(*ast.StarExpr); ok { + if _, ok := star.X.(*ast.CallExpr); ok { + // A call may return a pointer to a zero value. + return nil + } + } + return lockPath(f.pkg.typesPkg, f.pkg.types[x].Type) +} + +// lockPath returns a typePath describing the location of a lock value +// contained in typ. If there is no contained lock, it returns nil. +func lockPath(tpkg *types.Package, typ types.Type) typePath { + if typ == nil { + return nil + } + + // We're only interested in the case in which the underlying + // type is a struct. (Interfaces and pointers are safe to copy.) + styp, ok := typ.Underlying().(*types.Struct) + if !ok { + return nil + } + + // We're looking for cases in which a reference to this type + // can be locked, but a value cannot. This differentiates + // embedded interfaces from embedded values. + if plock := types.NewMethodSet(types.NewPointer(typ)).Lookup(tpkg, "Lock"); plock != nil { + if lock := types.NewMethodSet(typ).Lookup(tpkg, "Lock"); lock == nil { + return []types.Type{typ} + } + } + + nfields := styp.NumFields() + for i := 0; i < nfields; i++ { + ftyp := styp.Field(i).Type() + subpath := lockPath(tpkg, ftyp) + if subpath != nil { + return append(subpath, typ) + } + } + + return nil +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/deadcode.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/deadcode.go new file mode 100644 index 000000000..b1077aef3 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/deadcode.go @@ -0,0 +1,298 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Check for syntactically unreachable code. + +package main + +import ( + "go/ast" + "go/token" +) + +func init() { + register("unreachable", + "check for unreachable code", + checkUnreachable, + funcDecl, funcLit) +} + +type deadState struct { + f *File + hasBreak map[ast.Stmt]bool + hasGoto map[string]bool + labels map[string]ast.Stmt + breakTarget ast.Stmt + + reachable bool +} + +// checkUnreachable checks a function body for dead code. +// +// TODO(adonovan): use the new cfg package, which is more precise. +func checkUnreachable(f *File, node ast.Node) { + var body *ast.BlockStmt + switch n := node.(type) { + case *ast.FuncDecl: + body = n.Body + case *ast.FuncLit: + body = n.Body + } + if body == nil { + return + } + + d := &deadState{ + f: f, + hasBreak: make(map[ast.Stmt]bool), + hasGoto: make(map[string]bool), + labels: make(map[string]ast.Stmt), + } + + d.findLabels(body) + + d.reachable = true + d.findDead(body) +} + +// findLabels gathers information about the labels defined and used by stmt +// and about which statements break, whether a label is involved or not. +func (d *deadState) findLabels(stmt ast.Stmt) { + switch x := stmt.(type) { + default: + d.f.Warnf(x.Pos(), "internal error in findLabels: unexpected statement %T", x) + + case *ast.AssignStmt, + *ast.BadStmt, + *ast.DeclStmt, + *ast.DeferStmt, + *ast.EmptyStmt, + *ast.ExprStmt, + *ast.GoStmt, + *ast.IncDecStmt, + *ast.ReturnStmt, + *ast.SendStmt: + // no statements inside + + case *ast.BlockStmt: + for _, stmt := range x.List { + d.findLabels(stmt) + } + + case *ast.BranchStmt: + switch x.Tok { + case token.GOTO: + if x.Label != nil { + d.hasGoto[x.Label.Name] = true + } + + case token.BREAK: + stmt := d.breakTarget + if x.Label != nil { + stmt = d.labels[x.Label.Name] + } + if stmt != nil { + d.hasBreak[stmt] = true + } + } + + case *ast.IfStmt: + d.findLabels(x.Body) + if x.Else != nil { + d.findLabels(x.Else) + } + + case *ast.LabeledStmt: + d.labels[x.Label.Name] = x.Stmt + d.findLabels(x.Stmt) + + // These cases are all the same, but the x.Body only works + // when the specific type of x is known, so the cases cannot + // be merged. + case *ast.ForStmt: + outer := d.breakTarget + d.breakTarget = x + d.findLabels(x.Body) + d.breakTarget = outer + + case *ast.RangeStmt: + outer := d.breakTarget + d.breakTarget = x + d.findLabels(x.Body) + d.breakTarget = outer + + case *ast.SelectStmt: + outer := d.breakTarget + d.breakTarget = x + d.findLabels(x.Body) + d.breakTarget = outer + + case *ast.SwitchStmt: + outer := d.breakTarget + d.breakTarget = x + d.findLabels(x.Body) + d.breakTarget = outer + + case *ast.TypeSwitchStmt: + outer := d.breakTarget + d.breakTarget = x + d.findLabels(x.Body) + d.breakTarget = outer + + case *ast.CommClause: + for _, stmt := range x.Body { + d.findLabels(stmt) + } + + case *ast.CaseClause: + for _, stmt := range x.Body { + d.findLabels(stmt) + } + } +} + +// findDead walks the statement looking for dead code. +// If d.reachable is false on entry, stmt itself is dead. +// When findDead returns, d.reachable tells whether the +// statement following stmt is reachable. +func (d *deadState) findDead(stmt ast.Stmt) { + // Is this a labeled goto target? + // If so, assume it is reachable due to the goto. + // This is slightly conservative, in that we don't + // check that the goto is reachable, so + // L: goto L + // will not provoke a warning. + // But it's good enough. + if x, isLabel := stmt.(*ast.LabeledStmt); isLabel && d.hasGoto[x.Label.Name] { + d.reachable = true + } + + if !d.reachable { + switch stmt.(type) { + case *ast.EmptyStmt: + // do not warn about unreachable empty statements + default: + d.f.Bad(stmt.Pos(), "unreachable code") + d.reachable = true // silence error about next statement + } + } + + switch x := stmt.(type) { + default: + d.f.Warnf(x.Pos(), "internal error in findDead: unexpected statement %T", x) + + case *ast.AssignStmt, + *ast.BadStmt, + *ast.DeclStmt, + *ast.DeferStmt, + *ast.EmptyStmt, + *ast.GoStmt, + *ast.IncDecStmt, + *ast.SendStmt: + // no control flow + + case *ast.BlockStmt: + for _, stmt := range x.List { + d.findDead(stmt) + } + + case *ast.BranchStmt: + switch x.Tok { + case token.BREAK, token.GOTO, token.FALLTHROUGH: + d.reachable = false + case token.CONTINUE: + // NOTE: We accept "continue" statements as terminating. + // They are not necessary in the spec definition of terminating, + // because a continue statement cannot be the final statement + // before a return. But for the more general problem of syntactically + // identifying dead code, continue redirects control flow just + // like the other terminating statements. + d.reachable = false + } + + case *ast.ExprStmt: + // Call to panic? + call, ok := x.X.(*ast.CallExpr) + if ok { + name, ok := call.Fun.(*ast.Ident) + if ok && name.Name == "panic" && name.Obj == nil { + d.reachable = false + } + } + + case *ast.ForStmt: + d.findDead(x.Body) + d.reachable = x.Cond != nil || d.hasBreak[x] + + case *ast.IfStmt: + d.findDead(x.Body) + if x.Else != nil { + r := d.reachable + d.reachable = true + d.findDead(x.Else) + d.reachable = d.reachable || r + } else { + // might not have executed if statement + d.reachable = true + } + + case *ast.LabeledStmt: + d.findDead(x.Stmt) + + case *ast.RangeStmt: + d.findDead(x.Body) + d.reachable = true + + case *ast.ReturnStmt: + d.reachable = false + + case *ast.SelectStmt: + // NOTE: Unlike switch and type switch below, we don't care + // whether a select has a default, because a select without a + // default blocks until one of the cases can run. That's different + // from a switch without a default, which behaves like it has + // a default with an empty body. + anyReachable := false + for _, comm := range x.Body.List { + d.reachable = true + for _, stmt := range comm.(*ast.CommClause).Body { + d.findDead(stmt) + } + anyReachable = anyReachable || d.reachable + } + d.reachable = anyReachable || d.hasBreak[x] + + case *ast.SwitchStmt: + anyReachable := false + hasDefault := false + for _, cas := range x.Body.List { + cc := cas.(*ast.CaseClause) + if cc.List == nil { + hasDefault = true + } + d.reachable = true + for _, stmt := range cc.Body { + d.findDead(stmt) + } + anyReachable = anyReachable || d.reachable + } + d.reachable = anyReachable || d.hasBreak[x] || !hasDefault + + case *ast.TypeSwitchStmt: + anyReachable := false + hasDefault := false + for _, cas := range x.Body.List { + cc := cas.(*ast.CaseClause) + if cc.List == nil { + hasDefault = true + } + d.reachable = true + for _, stmt := range cc.Body { + d.findDead(stmt) + } + anyReachable = anyReachable || d.reachable + } + d.reachable = anyReachable || d.hasBreak[x] || !hasDefault + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/doc.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/doc.go new file mode 100644 index 000000000..69d5f9cc7 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/doc.go @@ -0,0 +1,205 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Vet examines Go source code and reports suspicious constructs, such as Printf +calls whose arguments do not align with the format string. Vet uses heuristics +that do not guarantee all reports are genuine problems, but it can find errors +not caught by the compilers. + +It can be invoked three ways: + +By package, from the go tool: + go vet package/path/name +vets the package whose path is provided. + +By files: + go tool vet source/directory/*.go +vets the files named, all of which must be in the same package. + +By directory: + go tool vet source/directory +recursively descends the directory, vetting each package it finds. + +Vet's exit code is 2 for erroneous invocation of the tool, 1 if a +problem was reported, and 0 otherwise. Note that the tool does not +check every possible problem and depends on unreliable heuristics +so it should be used as guidance only, not as a firm indicator of +program correctness. + +By default the -all flag is set so all checks are performed. +If any flags are explicitly set to true, only those tests are run. Conversely, if +any flag is explicitly set to false, only those tests are disabled. Thus -printf=true +runs the printf check, -printf=false runs all checks except the printf check. + +Available checks: + +Assembly declarations + +Flag: -asmdecl + +Mismatches between assembly files and Go function declarations. + +Useless assignments + +Flag: -assign + +Check for useless assignments. + +Atomic mistakes + +Flag: -atomic + +Common mistaken usages of the sync/atomic package. + +Boolean conditions + +Flag: -bool + +Mistakes involving boolean operators. + +Build tags + +Flag: -buildtags + +Badly formed or misplaced +build tags. + +Invalid uses of cgo + +Flag: -cgocall + +Detect some violations of the cgo pointer passing rules. + +Unkeyed composite literals + +Flag: -composites + +Composite struct literals that do not use the field-keyed syntax. + +Copying locks + +Flag: -copylocks + +Locks that are erroneously passed by value. + +Tests, benchmarks and documentation examples + +Flag: -tests + +Mistakes involving tests including functions with incorrect names or signatures +and example tests that document identifiers not in the package. + +Failure to call the cancelation function returned by context.WithCancel. + +Flag: -lostcancel + +The cancelation function returned by context.WithCancel, WithTimeout, +and WithDeadline must be called or the new context will remain live +until its parent context is cancelled. +(The background context is never cancelled.) + +Methods + +Flag: -methods + +Non-standard signatures for methods with familiar names, including: + Format GobEncode GobDecode MarshalJSON MarshalXML + Peek ReadByte ReadFrom ReadRune Scan Seek + UnmarshalJSON UnreadByte UnreadRune WriteByte + WriteTo + +Nil function comparison + +Flag: -nilfunc + +Comparisons between functions and nil. + +Printf family + +Flag: -printf + +Suspicious calls to functions in the Printf family, including any functions +with these names, disregarding case: + Print Printf Println + Fprint Fprintf Fprintln + Sprint Sprintf Sprintln + Error Errorf + Fatal Fatalf + Log Logf + Panic Panicf Panicln +The -printfuncs flag can be used to redefine this list. +If the function name ends with an 'f', the function is assumed to take +a format descriptor string in the manner of fmt.Printf. If not, vet +complains about arguments that look like format descriptor strings. + +It also checks for errors such as using a Writer as the first argument of +Printf. + +Struct tags + +Range loop variables + +Flag: -rangeloops + +Incorrect uses of range loop variables in closures. + +Shadowed variables + +Flag: -shadow=false (experimental; must be set explicitly) + +Variables that may have been unintentionally shadowed. + +Shifts + +Flag: -shift + +Shifts equal to or longer than the variable's length. + +Flag: -structtags + +Struct tags that do not follow the format understood by reflect.StructTag.Get. +Well-known encoding struct tags (json, xml) used with unexported fields. + +Unreachable code + +Flag: -unreachable + +Unreachable code. + +Misuse of unsafe Pointers + +Flag: -unsafeptr + +Likely incorrect uses of unsafe.Pointer to convert integers to pointers. +A conversion from uintptr to unsafe.Pointer is invalid if it implies that +there is a uintptr-typed word in memory that holds a pointer value, +because that word will be invisible to stack copying and to the garbage +collector. + +Unused result of certain function calls + +Flag: -unusedresult + +Calls to well-known functions and methods that return a value that is +discarded. By default, this includes functions like fmt.Errorf and +fmt.Sprintf and methods like String and Error. The flags -unusedfuncs +and -unusedstringmethods control the set. + +Other flags + +These flags configure the behavior of vet: + + -all (default true) + Enable all non-experimental checks. + -v + Verbose mode + -printfuncs + A comma-separated list of print-like function names + to supplement the standard list. + For more information, see the discussion of the -printf flag. + -shadowstrict + Whether to be strict about shadowing; can be noisy. +*/ +package main diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/internal/cfg/builder.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/internal/cfg/builder.go new file mode 100644 index 000000000..da1cc7e63 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/internal/cfg/builder.go @@ -0,0 +1,512 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cfg + +// This file implements the CFG construction pass. + +import ( + "fmt" + "go/ast" + "go/token" +) + +type builder struct { + cfg *CFG + mayReturn func(*ast.CallExpr) bool + current *Block + lblocks map[*ast.Object]*lblock // labeled blocks + targets *targets // linked stack of branch targets +} + +func (b *builder) stmt(_s ast.Stmt) { + // The label of the current statement. If non-nil, its _goto + // target is always set; its _break and _continue are set only + // within the body of switch/typeswitch/select/for/range. + // It is effectively an additional default-nil parameter of stmt(). + var label *lblock +start: + switch s := _s.(type) { + case *ast.BadStmt, + *ast.SendStmt, + *ast.IncDecStmt, + *ast.GoStmt, + *ast.DeferStmt, + *ast.EmptyStmt, + *ast.AssignStmt: + // No effect on control flow. + b.add(s) + + case *ast.ExprStmt: + b.add(s) + if call, ok := s.X.(*ast.CallExpr); ok && !b.mayReturn(call) { + // Calls to panic, os.Exit, etc, never return. + b.current = b.newUnreachableBlock("unreachable.call") + } + + case *ast.DeclStmt: + // Treat each var ValueSpec as a separate statement. + d := s.Decl.(*ast.GenDecl) + if d.Tok == token.VAR { + for _, spec := range d.Specs { + if spec, ok := spec.(*ast.ValueSpec); ok { + b.add(spec) + } + } + } + + case *ast.LabeledStmt: + label = b.labeledBlock(s.Label) + b.jump(label._goto) + b.current = label._goto + _s = s.Stmt + goto start // effectively: tailcall stmt(g, s.Stmt, label) + + case *ast.ReturnStmt: + b.add(s) + b.current = b.newUnreachableBlock("unreachable.return") + + case *ast.BranchStmt: + var block *Block + switch s.Tok { + case token.BREAK: + if s.Label != nil { + if lb := b.labeledBlock(s.Label); lb != nil { + block = lb._break + } + } else { + for t := b.targets; t != nil && block == nil; t = t.tail { + block = t._break + } + } + + case token.CONTINUE: + if s.Label != nil { + if lb := b.labeledBlock(s.Label); lb != nil { + block = lb._continue + } + } else { + for t := b.targets; t != nil && block == nil; t = t.tail { + block = t._continue + } + } + + case token.FALLTHROUGH: + for t := b.targets; t != nil; t = t.tail { + block = t._fallthrough + } + + case token.GOTO: + if s.Label != nil { + block = b.labeledBlock(s.Label)._goto + } + } + if block == nil { + block = b.newBlock("undefined.branch") + } + b.jump(block) + b.current = b.newUnreachableBlock("unreachable.branch") + + case *ast.BlockStmt: + b.stmtList(s.List) + + case *ast.IfStmt: + if s.Init != nil { + b.stmt(s.Init) + } + then := b.newBlock("if.then") + done := b.newBlock("if.done") + _else := done + if s.Else != nil { + _else = b.newBlock("if.else") + } + b.add(s.Cond) + b.ifelse(then, _else) + b.current = then + b.stmt(s.Body) + b.jump(done) + + if s.Else != nil { + b.current = _else + b.stmt(s.Else) + b.jump(done) + } + + b.current = done + + case *ast.SwitchStmt: + b.switchStmt(s, label) + + case *ast.TypeSwitchStmt: + b.typeSwitchStmt(s, label) + + case *ast.SelectStmt: + b.selectStmt(s, label) + + case *ast.ForStmt: + b.forStmt(s, label) + + case *ast.RangeStmt: + b.rangeStmt(s, label) + + default: + panic(fmt.Sprintf("unexpected statement kind: %T", s)) + } +} + +func (b *builder) stmtList(list []ast.Stmt) { + for _, s := range list { + b.stmt(s) + } +} + +func (b *builder) switchStmt(s *ast.SwitchStmt, label *lblock) { + if s.Init != nil { + b.stmt(s.Init) + } + if s.Tag != nil { + b.add(s.Tag) + } + done := b.newBlock("switch.done") + if label != nil { + label._break = done + } + // We pull the default case (if present) down to the end. + // But each fallthrough label must point to the next + // body block in source order, so we preallocate a + // body block (fallthru) for the next case. + // Unfortunately this makes for a confusing block order. + var defaultBody *[]ast.Stmt + var defaultFallthrough *Block + var fallthru, defaultBlock *Block + ncases := len(s.Body.List) + for i, clause := range s.Body.List { + body := fallthru + if body == nil { + body = b.newBlock("switch.body") // first case only + } + + // Preallocate body block for the next case. + fallthru = done + if i+1 < ncases { + fallthru = b.newBlock("switch.body") + } + + cc := clause.(*ast.CaseClause) + if cc.List == nil { + // Default case. + defaultBody = &cc.Body + defaultFallthrough = fallthru + defaultBlock = body + continue + } + + var nextCond *Block + for _, cond := range cc.List { + nextCond = b.newBlock("switch.next") + b.add(cond) // one half of the tag==cond condition + b.ifelse(body, nextCond) + b.current = nextCond + } + b.current = body + b.targets = &targets{ + tail: b.targets, + _break: done, + _fallthrough: fallthru, + } + b.stmtList(cc.Body) + b.targets = b.targets.tail + b.jump(done) + b.current = nextCond + } + if defaultBlock != nil { + b.jump(defaultBlock) + b.current = defaultBlock + b.targets = &targets{ + tail: b.targets, + _break: done, + _fallthrough: defaultFallthrough, + } + b.stmtList(*defaultBody) + b.targets = b.targets.tail + } + b.jump(done) + b.current = done +} + +func (b *builder) typeSwitchStmt(s *ast.TypeSwitchStmt, label *lblock) { + if s.Init != nil { + b.stmt(s.Init) + } + if s.Assign != nil { + b.add(s.Assign) + } + + done := b.newBlock("typeswitch.done") + if label != nil { + label._break = done + } + var default_ *ast.CaseClause + for _, clause := range s.Body.List { + cc := clause.(*ast.CaseClause) + if cc.List == nil { + default_ = cc + continue + } + body := b.newBlock("typeswitch.body") + var next *Block + for _, casetype := range cc.List { + next = b.newBlock("typeswitch.next") + // casetype is a type, so don't call b.add(casetype). + // This block logically contains a type assertion, + // x.(casetype), but it's unclear how to represent x. + _ = casetype + b.ifelse(body, next) + b.current = next + } + b.current = body + b.typeCaseBody(cc, done) + b.current = next + } + if default_ != nil { + b.typeCaseBody(default_, done) + } else { + b.jump(done) + } + b.current = done +} + +func (b *builder) typeCaseBody(cc *ast.CaseClause, done *Block) { + b.targets = &targets{ + tail: b.targets, + _break: done, + } + b.stmtList(cc.Body) + b.targets = b.targets.tail + b.jump(done) +} + +func (b *builder) selectStmt(s *ast.SelectStmt, label *lblock) { + // First evaluate channel expressions. + // TODO(adonovan): fix: evaluate only channel exprs here. + for _, clause := range s.Body.List { + if comm := clause.(*ast.CommClause).Comm; comm != nil { + b.stmt(comm) + } + } + + done := b.newBlock("select.done") + if label != nil { + label._break = done + } + + var defaultBody *[]ast.Stmt + for _, cc := range s.Body.List { + clause := cc.(*ast.CommClause) + if clause.Comm == nil { + defaultBody = &clause.Body + continue + } + body := b.newBlock("select.body") + next := b.newBlock("select.next") + b.ifelse(body, next) + b.current = body + b.targets = &targets{ + tail: b.targets, + _break: done, + } + switch comm := clause.Comm.(type) { + case *ast.ExprStmt: // <-ch + // nop + case *ast.AssignStmt: // x := <-states[state].Chan + b.add(comm.Lhs[0]) + } + b.stmtList(clause.Body) + b.targets = b.targets.tail + b.jump(done) + b.current = next + } + if defaultBody != nil { + b.targets = &targets{ + tail: b.targets, + _break: done, + } + b.stmtList(*defaultBody) + b.targets = b.targets.tail + b.jump(done) + } + b.current = done +} + +func (b *builder) forStmt(s *ast.ForStmt, label *lblock) { + // ...init... + // jump loop + // loop: + // if cond goto body else done + // body: + // ...body... + // jump post + // post: (target of continue) + // ...post... + // jump loop + // done: (target of break) + if s.Init != nil { + b.stmt(s.Init) + } + body := b.newBlock("for.body") + done := b.newBlock("for.done") // target of 'break' + loop := body // target of back-edge + if s.Cond != nil { + loop = b.newBlock("for.loop") + } + cont := loop // target of 'continue' + if s.Post != nil { + cont = b.newBlock("for.post") + } + if label != nil { + label._break = done + label._continue = cont + } + b.jump(loop) + b.current = loop + if loop != body { + b.add(s.Cond) + b.ifelse(body, done) + b.current = body + } + b.targets = &targets{ + tail: b.targets, + _break: done, + _continue: cont, + } + b.stmt(s.Body) + b.targets = b.targets.tail + b.jump(cont) + + if s.Post != nil { + b.current = cont + b.stmt(s.Post) + b.jump(loop) // back-edge + } + b.current = done +} + +func (b *builder) rangeStmt(s *ast.RangeStmt, label *lblock) { + b.add(s.X) + + if s.Key != nil { + b.add(s.Key) + } + if s.Value != nil { + b.add(s.Value) + } + + // ... + // loop: (target of continue) + // if ... goto body else done + // body: + // ... + // jump loop + // done: (target of break) + + loop := b.newBlock("range.loop") + b.jump(loop) + b.current = loop + + body := b.newBlock("range.body") + done := b.newBlock("range.done") + b.ifelse(body, done) + b.current = body + + if label != nil { + label._break = done + label._continue = loop + } + b.targets = &targets{ + tail: b.targets, + _break: done, + _continue: loop, + } + b.stmt(s.Body) + b.targets = b.targets.tail + b.jump(loop) // back-edge + b.current = done +} + +// -------- helpers -------- + +// Destinations associated with unlabeled for/switch/select stmts. +// We push/pop one of these as we enter/leave each construct and for +// each BranchStmt we scan for the innermost target of the right type. +// +type targets struct { + tail *targets // rest of stack + _break *Block + _continue *Block + _fallthrough *Block +} + +// Destinations associated with a labeled block. +// We populate these as labels are encountered in forward gotos or +// labeled statements. +// +type lblock struct { + _goto *Block + _break *Block + _continue *Block +} + +// labeledBlock returns the branch target associated with the +// specified label, creating it if needed. +// +func (b *builder) labeledBlock(label *ast.Ident) *lblock { + lb := b.lblocks[label.Obj] + if lb == nil { + lb = &lblock{_goto: b.newBlock(label.Name)} + if b.lblocks == nil { + b.lblocks = make(map[*ast.Object]*lblock) + } + b.lblocks[label.Obj] = lb + } + return lb +} + +// newBlock appends a new unconnected basic block to b.cfg's block +// slice and returns it. +// It does not automatically become the current block. +// comment is an optional string for more readable debugging output. +func (b *builder) newBlock(comment string) *Block { + g := b.cfg + block := &Block{ + index: int32(len(g.Blocks)), + comment: comment, + } + block.Succs = block.succs2[:0] + g.Blocks = append(g.Blocks, block) + return block +} + +func (b *builder) newUnreachableBlock(comment string) *Block { + block := b.newBlock(comment) + block.unreachable = true + return block +} + +func (b *builder) add(n ast.Node) { + b.current.Nodes = append(b.current.Nodes, n) +} + +// jump adds an edge from the current block to the target block, +// and sets b.current to nil. +func (b *builder) jump(target *Block) { + b.current.Succs = append(b.current.Succs, target) + b.current = nil +} + +// ifelse emits edges from the current block to the t and f blocks, +// and sets b.current to nil. +func (b *builder) ifelse(t, f *Block) { + b.current.Succs = append(b.current.Succs, t, f) + b.current = nil +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/internal/cfg/cfg.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/internal/cfg/cfg.go new file mode 100644 index 000000000..e4d5bfe5d --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/internal/cfg/cfg.go @@ -0,0 +1,142 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package constructs a simple control-flow graph (CFG) of the +// statements and expressions within a single function. +// +// Use cfg.New to construct the CFG for a function body. +// +// The blocks of the CFG contain all the function's non-control +// statements. The CFG does not contain control statements such as If, +// Switch, Select, and Branch, but does contain their subexpressions. +// For example, this source code: +// +// if x := f(); x != nil { +// T() +// } else { +// F() +// } +// +// produces this CFG: +// +// 1: x := f() +// x != nil +// succs: 2, 3 +// 2: T() +// succs: 4 +// 3: F() +// succs: 4 +// 4: +// +// The CFG does contain Return statements; even implicit returns are +// materialized (at the position of the function's closing brace). +// +// The CFG does not record conditions associated with conditional branch +// edges, nor the short-circuit semantics of the && and || operators, +// nor abnormal control flow caused by panic. If you need this +// information, use golang.org/x/tools/go/ssa instead. +// +package cfg + +// Although the vet tool has type information, it is often extremely +// fragmentary, so for simplicity this package does not depend on +// go/types. Consequently control-flow conditions are ignored even +// when constant, and "mayReturn" information must be provided by the +// client. +import ( + "bytes" + "fmt" + "go/ast" + "go/format" + "go/token" +) + +// A CFG represents the control-flow graph of a single function. +// +// The entry point is Blocks[0]; there may be multiple return blocks. +type CFG struct { + Blocks []*Block // block[0] is entry; order otherwise undefined +} + +// A Block represents a basic block: a list of statements and +// expressions that are always evaluated sequentially. +// +// A block may have 0-2 successors: zero for a return block or a block +// that calls a function such as panic that never returns; one for a +// normal (jump) block; and two for a conditional (if) block. +type Block struct { + Nodes []ast.Node // statements, expressions, and ValueSpecs + Succs []*Block // successor nodes in the graph + + comment string // for debugging + index int32 // index within CFG.Blocks + unreachable bool // is block of stmts following return/panic/for{} + succs2 [2]*Block // underlying array for Succs +} + +// New returns a new control-flow graph for the specified function body, +// which must be non-nil. +// +// The CFG builder calls mayReturn to determine whether a given function +// call may return. For example, calls to panic, os.Exit, and log.Fatal +// do not return, so the builder can remove infeasible graph edges +// following such calls. The builder calls mayReturn only for a +// CallExpr beneath an ExprStmt. +func New(body *ast.BlockStmt, mayReturn func(*ast.CallExpr) bool) *CFG { + b := builder{ + mayReturn: mayReturn, + cfg: new(CFG), + } + b.current = b.newBlock("entry") + b.stmt(body) + + // Does control fall off the end of the function's body? + // Make implicit return explicit. + if b.current != nil && !b.current.unreachable { + b.add(&ast.ReturnStmt{ + Return: body.End() - 1, + }) + } + + return b.cfg +} + +func (b *Block) String() string { + return fmt.Sprintf("block %d (%s)", b.index, b.comment) +} + +// Return returns the return statement at the end of this block if present, nil otherwise. +func (b *Block) Return() (ret *ast.ReturnStmt) { + if len(b.Nodes) > 0 { + ret, _ = b.Nodes[len(b.Nodes)-1].(*ast.ReturnStmt) + } + return +} + +// Format formats the control-flow graph for ease of debugging. +func (g *CFG) Format(fset *token.FileSet) string { + var buf bytes.Buffer + for _, b := range g.Blocks { + fmt.Fprintf(&buf, ".%d: # %s\n", b.index, b.comment) + for _, n := range b.Nodes { + fmt.Fprintf(&buf, "\t%s\n", formatNode(fset, n)) + } + if len(b.Succs) > 0 { + fmt.Fprintf(&buf, "\tsuccs:") + for _, succ := range b.Succs { + fmt.Fprintf(&buf, " %d", succ.index) + } + buf.WriteByte('\n') + } + buf.WriteByte('\n') + } + return buf.String() +} + +func formatNode(fset *token.FileSet, n ast.Node) string { + var buf bytes.Buffer + format.Node(&buf, fset, n) + // Indent secondary lines by a tab. + return string(bytes.Replace(buf.Bytes(), []byte("\n"), []byte("\n\t"), -1)) +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/internal/whitelist/whitelist.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/internal/whitelist/whitelist.go new file mode 100644 index 000000000..fdd65d373 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/internal/whitelist/whitelist.go @@ -0,0 +1,28 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package whitelist defines exceptions for the vet tool. +package whitelist + +// UnkeyedLiteral is a white list of types in the standard packages +// that are used with unkeyed literals we deem to be acceptable. +var UnkeyedLiteral = map[string]bool{ + // These image and image/color struct types are frozen. We will never add fields to them. + "image/color.Alpha16": true, + "image/color.Alpha": true, + "image/color.CMYK": true, + "image/color.Gray16": true, + "image/color.Gray": true, + "image/color.NRGBA64": true, + "image/color.NRGBA": true, + "image/color.NYCbCrA": true, + "image/color.RGBA64": true, + "image/color.RGBA": true, + "image/color.YCbCr": true, + "image.Point": true, + "image.Rectangle": true, + "image.Uniform": true, + + "unicode.Range16": true, +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/lostcancel.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/lostcancel.go new file mode 100644 index 000000000..adaff23e1 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/lostcancel.go @@ -0,0 +1,318 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "github.com/dnephin/govet/internal/cfg" + "fmt" + "go/ast" + "go/types" + "strconv" +) + +func init() { + register("lostcancel", + "check for failure to call cancelation function returned by context.WithCancel", + checkLostCancel, + funcDecl, funcLit) +} + +const debugLostCancel = false + +var contextPackage = "context" + +// checkLostCancel reports a failure to the call the cancel function +// returned by context.WithCancel, either because the variable was +// assigned to the blank identifier, or because there exists a +// control-flow path from the call to a return statement and that path +// does not "use" the cancel function. Any reference to the variable +// counts as a use, even within a nested function literal. +// +// checkLostCancel analyzes a single named or literal function. +func checkLostCancel(f *File, node ast.Node) { + // Fast path: bypass check if file doesn't use context.WithCancel. + if !hasImport(f.file, contextPackage) { + return + } + + // Maps each cancel variable to its defining ValueSpec/AssignStmt. + cancelvars := make(map[*types.Var]ast.Node) + + // Find the set of cancel vars to analyze. + stack := make([]ast.Node, 0, 32) + ast.Inspect(node, func(n ast.Node) bool { + switch n.(type) { + case *ast.FuncLit: + if len(stack) > 0 { + return false // don't stray into nested functions + } + case nil: + stack = stack[:len(stack)-1] // pop + return true + } + stack = append(stack, n) // push + + // Look for [{AssignStmt,ValueSpec} CallExpr SelectorExpr]: + // + // ctx, cancel := context.WithCancel(...) + // ctx, cancel = context.WithCancel(...) + // var ctx, cancel = context.WithCancel(...) + // + if isContextWithCancel(f, n) && isCall(stack[len(stack)-2]) { + var id *ast.Ident // id of cancel var + stmt := stack[len(stack)-3] + switch stmt := stmt.(type) { + case *ast.ValueSpec: + if len(stmt.Names) > 1 { + id = stmt.Names[1] + } + case *ast.AssignStmt: + if len(stmt.Lhs) > 1 { + id, _ = stmt.Lhs[1].(*ast.Ident) + } + } + if id != nil { + if id.Name == "_" { + f.Badf(id.Pos(), "the cancel function returned by context.%s should be called, not discarded, to avoid a context leak", + n.(*ast.SelectorExpr).Sel.Name) + } else if v, ok := f.pkg.uses[id].(*types.Var); ok { + cancelvars[v] = stmt + } else if v, ok := f.pkg.defs[id].(*types.Var); ok { + cancelvars[v] = stmt + } + } + } + + return true + }) + + if len(cancelvars) == 0 { + return // no need to build CFG + } + + // Tell the CFG builder which functions never return. + info := &types.Info{Uses: f.pkg.uses, Selections: f.pkg.selectors} + mayReturn := func(call *ast.CallExpr) bool { + name := callName(info, call) + return !noReturnFuncs[name] + } + + // Build the CFG. + var g *cfg.CFG + var sig *types.Signature + switch node := node.(type) { + case *ast.FuncDecl: + sig, _ = f.pkg.defs[node.Name].Type().(*types.Signature) + g = cfg.New(node.Body, mayReturn) + case *ast.FuncLit: + sig, _ = f.pkg.types[node.Type].Type.(*types.Signature) + g = cfg.New(node.Body, mayReturn) + } + + // Print CFG. + if debugLostCancel { + fmt.Println(g.Format(f.fset)) + } + + // Examine the CFG for each variable in turn. + // (It would be more efficient to analyze all cancelvars in a + // single pass over the AST, but seldom is there more than one.) + for v, stmt := range cancelvars { + if ret := lostCancelPath(f, g, v, stmt, sig); ret != nil { + lineno := f.fset.Position(stmt.Pos()).Line + f.Badf(stmt.Pos(), "the %s function is not used on all paths (possible context leak)", v.Name()) + f.Badf(ret.Pos(), "this return statement may be reached without using the %s var defined on line %d", v.Name(), lineno) + } + } +} + +func isCall(n ast.Node) bool { _, ok := n.(*ast.CallExpr); return ok } + +func hasImport(f *ast.File, path string) bool { + for _, imp := range f.Imports { + v, _ := strconv.Unquote(imp.Path.Value) + if v == path { + return true + } + } + return false +} + +// isContextWithCancel reports whether n is one of the qualified identifiers +// context.With{Cancel,Timeout,Deadline}. +func isContextWithCancel(f *File, n ast.Node) bool { + if sel, ok := n.(*ast.SelectorExpr); ok { + switch sel.Sel.Name { + case "WithCancel", "WithTimeout", "WithDeadline": + if x, ok := sel.X.(*ast.Ident); ok { + if pkgname, ok := f.pkg.uses[x].(*types.PkgName); ok { + return pkgname.Imported().Path() == contextPackage + } + // Import failed, so we can't check package path. + // Just check the local package name (heuristic). + return x.Name == "context" + } + } + } + return false +} + +// lostCancelPath finds a path through the CFG, from stmt (which defines +// the 'cancel' variable v) to a return statement, that doesn't "use" v. +// If it finds one, it returns the return statement (which may be synthetic). +// sig is the function's type, if known. +func lostCancelPath(f *File, g *cfg.CFG, v *types.Var, stmt ast.Node, sig *types.Signature) *ast.ReturnStmt { + vIsNamedResult := sig != nil && tupleContains(sig.Results(), v) + + // uses reports whether stmts contain a "use" of variable v. + uses := func(f *File, v *types.Var, stmts []ast.Node) bool { + found := false + for _, stmt := range stmts { + ast.Inspect(stmt, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.Ident: + if f.pkg.uses[n] == v { + found = true + } + case *ast.ReturnStmt: + // A naked return statement counts as a use + // of the named result variables. + if n.Results == nil && vIsNamedResult { + found = true + } + } + return !found + }) + } + return found + } + + // blockUses computes "uses" for each block, caching the result. + memo := make(map[*cfg.Block]bool) + blockUses := func(f *File, v *types.Var, b *cfg.Block) bool { + res, ok := memo[b] + if !ok { + res = uses(f, v, b.Nodes) + memo[b] = res + } + return res + } + + // Find the var's defining block in the CFG, + // plus the rest of the statements of that block. + var defblock *cfg.Block + var rest []ast.Node +outer: + for _, b := range g.Blocks { + for i, n := range b.Nodes { + if n == stmt { + defblock = b + rest = b.Nodes[i+1:] + break outer + } + } + } + if defblock == nil { + panic("internal error: can't find defining block for cancel var") + } + + // Is v "used" in the remainder of its defining block? + if uses(f, v, rest) { + return nil + } + + // Does the defining block return without using v? + if ret := defblock.Return(); ret != nil { + return ret + } + + // Search the CFG depth-first for a path, from defblock to a + // return block, in which v is never "used". + seen := make(map[*cfg.Block]bool) + var search func(blocks []*cfg.Block) *ast.ReturnStmt + search = func(blocks []*cfg.Block) *ast.ReturnStmt { + for _, b := range blocks { + if !seen[b] { + seen[b] = true + + // Prune the search if the block uses v. + if blockUses(f, v, b) { + continue + } + + // Found path to return statement? + if ret := b.Return(); ret != nil { + if debugLostCancel { + fmt.Printf("found path to return in block %s\n", b) + } + return ret // found + } + + // Recur + if ret := search(b.Succs); ret != nil { + if debugLostCancel { + fmt.Printf(" from block %s\n", b) + } + return ret + } + } + } + return nil + } + return search(defblock.Succs) +} + +func tupleContains(tuple *types.Tuple, v *types.Var) bool { + for i := 0; i < tuple.Len(); i++ { + if tuple.At(i) == v { + return true + } + } + return false +} + +var noReturnFuncs = map[string]bool{ + "(*testing.common).FailNow": true, + "(*testing.common).Fatal": true, + "(*testing.common).Fatalf": true, + "(*testing.common).Skip": true, + "(*testing.common).SkipNow": true, + "(*testing.common).Skipf": true, + "log.Fatal": true, + "log.Fatalf": true, + "log.Fatalln": true, + "os.Exit": true, + "panic": true, + "runtime.Goexit": true, +} + +// callName returns the canonical name of the builtin, method, or +// function called by call, if known. +func callName(info *types.Info, call *ast.CallExpr) string { + switch fun := call.Fun.(type) { + case *ast.Ident: + // builtin, e.g. "panic" + if obj, ok := info.Uses[fun].(*types.Builtin); ok { + return obj.Name() + } + case *ast.SelectorExpr: + if sel, ok := info.Selections[fun]; ok && sel.Kind() == types.MethodVal { + // method call, e.g. "(*testing.common).Fatal" + meth := sel.Obj() + return fmt.Sprintf("(%s).%s", + meth.Type().(*types.Signature).Recv().Type(), + meth.Name()) + } + if obj, ok := info.Uses[fun.Sel]; ok { + // qualified identifier, e.g. "os.Exit" + return fmt.Sprintf("%s.%s", + obj.Pkg().Path(), + obj.Name()) + } + } + + // function with no name, or defined in missing imported package + return "" +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/main.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/main.go new file mode 100644 index 000000000..df6e0e16f --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/main.go @@ -0,0 +1,504 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Vet is a simple checker for static errors in Go source code. +// See doc.go for more information. +package main + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/printer" + "go/token" + "go/types" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" +) + +var ( + verbose = flag.Bool("v", false, "verbose") + tags = flag.String("tags", "", "comma-separated list of build tags to apply when parsing") + noRecurse = flag.Bool("no-recurse", false, "disable recursive directory walking") + tagList = []string{} // exploded version of tags flag; set in main +) + +var exitCode = 0 + +// "-all" flag enables all non-experimental checks +var all = triStateFlag("all", unset, "enable all non-experimental checks") + +// Flags to control which individual checks to perform. +var report = map[string]*triState{ + // Only unusual checks are written here. + // Most checks that operate during the AST walk are added by register. + "asmdecl": triStateFlag("asmdecl", unset, "check assembly against Go declarations"), + "buildtags": triStateFlag("buildtags", unset, "check that +build tags are valid"), +} + +// experimental records the flags enabling experimental features. These must be +// requested explicitly; they are not enabled by -all. +var experimental = map[string]bool{} + +// setTrueCount record how many flags are explicitly set to true. +var setTrueCount int + +// dirsRun and filesRun indicate whether the vet is applied to directory or +// file targets. The distinction affects which checks are run. +var dirsRun, filesRun bool + +// includesNonTest indicates whether the vet is applied to non-test targets. +// Certain checks are relevant only if they touch both test and non-test files. +var includesNonTest bool + +// A triState is a boolean that knows whether it has been set to either true or false. +// It is used to identify if a flag appears; the standard boolean flag cannot +// distinguish missing from unset. It also satisfies flag.Value. +type triState int + +const ( + unset triState = iota + setTrue + setFalse +) + +func triStateFlag(name string, value triState, usage string) *triState { + flag.Var(&value, name, usage) + return &value +} + +// triState implements flag.Value, flag.Getter, and flag.boolFlag. +// They work like boolean flags: we can say vet -printf as well as vet -printf=true +func (ts *triState) Get() interface{} { + return *ts == setTrue +} + +func (ts triState) isTrue() bool { + return ts == setTrue +} + +func (ts *triState) Set(value string) error { + b, err := strconv.ParseBool(value) + if err != nil { + return err + } + if b { + *ts = setTrue + setTrueCount++ + } else { + *ts = setFalse + } + return nil +} + +func (ts *triState) String() string { + switch *ts { + case unset: + return "true" // An unset flag will be set by -all, so defaults to true. + case setTrue: + return "true" + case setFalse: + return "false" + } + panic("not reached") +} + +func (ts triState) IsBoolFlag() bool { + return true +} + +// vet tells whether to report errors for the named check, a flag name. +func vet(name string) bool { + return report[name].isTrue() +} + +// setExit sets the value for os.Exit when it is called, later. It +// remembers the highest value. +func setExit(err int) { + if err > exitCode { + exitCode = err + } +} + +var ( + // Each of these vars has a corresponding case in (*File).Visit. + assignStmt *ast.AssignStmt + binaryExpr *ast.BinaryExpr + callExpr *ast.CallExpr + compositeLit *ast.CompositeLit + exprStmt *ast.ExprStmt + field *ast.Field + funcDecl *ast.FuncDecl + funcLit *ast.FuncLit + genDecl *ast.GenDecl + interfaceType *ast.InterfaceType + rangeStmt *ast.RangeStmt + returnStmt *ast.ReturnStmt + + // checkers is a two-level map. + // The outer level is keyed by a nil pointer, one of the AST vars above. + // The inner level is keyed by checker name. + checkers = make(map[ast.Node]map[string]func(*File, ast.Node)) +) + +func register(name, usage string, fn func(*File, ast.Node), types ...ast.Node) { + report[name] = triStateFlag(name, unset, usage) + for _, typ := range types { + m := checkers[typ] + if m == nil { + m = make(map[string]func(*File, ast.Node)) + checkers[typ] = m + } + m[name] = fn + } +} + +// Usage is a replacement usage function for the flags package. +func Usage() { + fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "\tvet [flags] directory...\n") + fmt.Fprintf(os.Stderr, "\tvet [flags] files... # Must be a single package\n") + fmt.Fprintf(os.Stderr, "By default, -all is set and all non-experimental checks are run.\n") + fmt.Fprintf(os.Stderr, "For more information run\n") + fmt.Fprintf(os.Stderr, "\tgo doc cmd/vet\n\n") + fmt.Fprintf(os.Stderr, "Flags:\n") + flag.PrintDefaults() + os.Exit(2) +} + +// File is a wrapper for the state of a file used in the parser. +// The parse tree walkers are all methods of this type. +type File struct { + pkg *Package + fset *token.FileSet + name string + content []byte + file *ast.File + b bytes.Buffer // for use by methods + + // Parsed package "foo" when checking package "foo_test" + basePkg *Package + + // The objects that are receivers of a "String() string" method. + // This is used by the recursiveStringer method in print.go. + stringers map[*ast.Object]bool + + // Registered checkers to run. + checkers map[ast.Node][]func(*File, ast.Node) +} + +func main() { + flag.Usage = Usage + flag.Parse() + + // If any flag is set, we run only those checks requested. + // If all flag is set true or if no flags are set true, set all the non-experimental ones + // not explicitly set (in effect, set the "-all" flag). + if setTrueCount == 0 || *all == setTrue { + for name, setting := range report { + if *setting == unset && !experimental[name] { + *setting = setTrue + } + } + } + + tagList = strings.Split(*tags, ",") + + initPrintFlags() + initUnusedFlags() + + if flag.NArg() == 0 { + Usage() + } + for _, name := range flag.Args() { + // Is it a directory? + fi, err := os.Stat(name) + if err != nil { + warnf("error walking tree: %s", err) + continue + } + if fi.IsDir() { + dirsRun = true + } else { + filesRun = true + if !strings.HasSuffix(name, "_test.go") { + includesNonTest = true + } + } + } + if dirsRun && filesRun { + Usage() + } + if dirsRun { + for _, name := range flag.Args() { + if *noRecurse { + doPackageDir(name) + } else { + walkDir(name) + } + } + os.Exit(exitCode) + } + if doPackage(".", flag.Args(), nil) == nil { + warnf("no files checked") + } + os.Exit(exitCode) +} + +// prefixDirectory places the directory name on the beginning of each name in the list. +func prefixDirectory(directory string, names []string) { + if directory != "." { + for i, name := range names { + names[i] = filepath.Join(directory, name) + } + } +} + +// doPackageDir analyzes the single package found in the directory, if there is one, +// plus a test package, if there is one. +func doPackageDir(directory string) { + context := build.Default + if len(context.BuildTags) != 0 { + warnf("build tags %s previously set", context.BuildTags) + } + context.BuildTags = append(tagList, context.BuildTags...) + + pkg, err := context.ImportDir(directory, 0) + if err != nil { + // If it's just that there are no go source files, that's fine. + if _, nogo := err.(*build.NoGoError); nogo { + return + } + // Non-fatal: we are doing a recursive walk and there may be other directories. + warnf("cannot process directory %s: %s", directory, err) + return + } + var names []string + names = append(names, pkg.GoFiles...) + names = append(names, pkg.CgoFiles...) + names = append(names, pkg.TestGoFiles...) // These are also in the "foo" package. + names = append(names, pkg.SFiles...) + prefixDirectory(directory, names) + basePkg := doPackage(directory, names, nil) + // Is there also a "foo_test" package? If so, do that one as well. + if len(pkg.XTestGoFiles) > 0 { + names = pkg.XTestGoFiles + prefixDirectory(directory, names) + doPackage(directory, names, basePkg) + } +} + +type Package struct { + path string + defs map[*ast.Ident]types.Object + uses map[*ast.Ident]types.Object + selectors map[*ast.SelectorExpr]*types.Selection + types map[ast.Expr]types.TypeAndValue + spans map[types.Object]Span + files []*File + typesPkg *types.Package +} + +// doPackage analyzes the single package constructed from the named files. +// It returns the parsed Package or nil if none of the files have been checked. +func doPackage(directory string, names []string, basePkg *Package) *Package { + var files []*File + var astFiles []*ast.File + fs := token.NewFileSet() + for _, name := range names { + data, err := ioutil.ReadFile(name) + if err != nil { + // Warn but continue to next package. + warnf("%s: %s", name, err) + return nil + } + checkBuildTag(name, data) + var parsedFile *ast.File + if strings.HasSuffix(name, ".go") { + parsedFile, err = parser.ParseFile(fs, name, data, 0) + if err != nil { + warnf("%s: %s", name, err) + return nil + } + astFiles = append(astFiles, parsedFile) + } + files = append(files, &File{fset: fs, content: data, name: name, file: parsedFile}) + } + if len(astFiles) == 0 { + return nil + } + pkg := new(Package) + pkg.path = astFiles[0].Name.Name + pkg.files = files + // Type check the package. + err := pkg.check(fs, astFiles) + if err != nil && *verbose { + warnf("%s", err) + } + + // Check. + chk := make(map[ast.Node][]func(*File, ast.Node)) + for typ, set := range checkers { + for name, fn := range set { + if vet(name) { + chk[typ] = append(chk[typ], fn) + } + } + } + for _, file := range files { + file.pkg = pkg + file.basePkg = basePkg + file.checkers = chk + if file.file != nil { + file.walkFile(file.name, file.file) + } + } + asmCheck(pkg) + return pkg +} + +func visit(path string, f os.FileInfo, err error) error { + if err != nil { + warnf("walk error: %s", err) + return err + } + // One package per directory. Ignore the files themselves. + if !f.IsDir() { + return nil + } + doPackageDir(path) + return nil +} + +func (pkg *Package) hasFileWithSuffix(suffix string) bool { + for _, f := range pkg.files { + if strings.HasSuffix(f.name, suffix) { + return true + } + } + return false +} + +// walkDir recursively walks the tree looking for Go packages. +func walkDir(root string) { + filepath.Walk(root, visit) +} + +// errorf formats the error to standard error, adding program +// identification and a newline, and exits. +func errorf(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, "vet: "+format+"\n", args...) + os.Exit(2) +} + +// warnf formats the error to standard error, adding program +// identification and a newline, but does not exit. +func warnf(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, "vet: "+format+"\n", args...) + setExit(1) +} + +// Println is fmt.Println guarded by -v. +func Println(args ...interface{}) { + if !*verbose { + return + } + fmt.Println(args...) +} + +// Printf is fmt.Printf guarded by -v. +func Printf(format string, args ...interface{}) { + if !*verbose { + return + } + fmt.Printf(format+"\n", args...) +} + +// Bad reports an error and sets the exit code.. +func (f *File) Bad(pos token.Pos, args ...interface{}) { + f.Warn(pos, args...) + setExit(1) +} + +// Badf reports a formatted error and sets the exit code. +func (f *File) Badf(pos token.Pos, format string, args ...interface{}) { + f.Warnf(pos, format, args...) + setExit(1) +} + +// loc returns a formatted representation of the position. +func (f *File) loc(pos token.Pos) string { + if pos == token.NoPos { + return "" + } + // Do not print columns. Because the pos often points to the start of an + // expression instead of the inner part with the actual error, the + // precision can mislead. + posn := f.fset.Position(pos) + return fmt.Sprintf("%s:%d", posn.Filename, posn.Line) +} + +// Warn reports an error but does not set the exit code. +func (f *File) Warn(pos token.Pos, args ...interface{}) { + fmt.Fprintf(os.Stderr, "%s: %s", f.loc(pos), fmt.Sprintln(args...)) +} + +// Warnf reports a formatted error but does not set the exit code. +func (f *File) Warnf(pos token.Pos, format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, "%s: %s\n", f.loc(pos), fmt.Sprintf(format, args...)) +} + +// walkFile walks the file's tree. +func (f *File) walkFile(name string, file *ast.File) { + Println("Checking file", name) + ast.Walk(f, file) +} + +// Visit implements the ast.Visitor interface. +func (f *File) Visit(node ast.Node) ast.Visitor { + var key ast.Node + switch node.(type) { + case *ast.AssignStmt: + key = assignStmt + case *ast.BinaryExpr: + key = binaryExpr + case *ast.CallExpr: + key = callExpr + case *ast.CompositeLit: + key = compositeLit + case *ast.ExprStmt: + key = exprStmt + case *ast.Field: + key = field + case *ast.FuncDecl: + key = funcDecl + case *ast.FuncLit: + key = funcLit + case *ast.GenDecl: + key = genDecl + case *ast.InterfaceType: + key = interfaceType + case *ast.RangeStmt: + key = rangeStmt + case *ast.ReturnStmt: + key = returnStmt + } + for _, fn := range f.checkers[key] { + fn(f, node) + } + return f +} + +// gofmt returns a string representation of the expression. +func (f *File) gofmt(x ast.Expr) string { + f.b.Reset() + printer.Fprint(&f.b, f.fset, x) + return f.b.String() +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/method.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/method.go new file mode 100644 index 000000000..8a554e152 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/method.go @@ -0,0 +1,182 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the code to check canonical methods. + +package main + +import ( + "fmt" + "go/ast" + "go/printer" + "strings" +) + +func init() { + register("methods", + "check that canonically named methods are canonically defined", + checkCanonicalMethod, + funcDecl, interfaceType) +} + +type MethodSig struct { + args []string + results []string +} + +// canonicalMethods lists the input and output types for Go methods +// that are checked using dynamic interface checks. Because the +// checks are dynamic, such methods would not cause a compile error +// if they have the wrong signature: instead the dynamic check would +// fail, sometimes mysteriously. If a method is found with a name listed +// here but not the input/output types listed here, vet complains. +// +// A few of the canonical methods have very common names. +// For example, a type might implement a Scan method that +// has nothing to do with fmt.Scanner, but we still want to check +// the methods that are intended to implement fmt.Scanner. +// To do that, the arguments that have a = prefix are treated as +// signals that the canonical meaning is intended: if a Scan +// method doesn't have a fmt.ScanState as its first argument, +// we let it go. But if it does have a fmt.ScanState, then the +// rest has to match. +var canonicalMethods = map[string]MethodSig{ + // "Flush": {{}, {"error"}}, // http.Flusher and jpeg.writer conflict + "Format": {[]string{"=fmt.State", "rune"}, []string{}}, // fmt.Formatter + "GobDecode": {[]string{"[]byte"}, []string{"error"}}, // gob.GobDecoder + "GobEncode": {[]string{}, []string{"[]byte", "error"}}, // gob.GobEncoder + "MarshalJSON": {[]string{}, []string{"[]byte", "error"}}, // json.Marshaler + "MarshalXML": {[]string{"*xml.Encoder", "xml.StartElement"}, []string{"error"}}, // xml.Marshaler + "Peek": {[]string{"=int"}, []string{"[]byte", "error"}}, // image.reader (matching bufio.Reader) + "ReadByte": {[]string{}, []string{"byte", "error"}}, // io.ByteReader + "ReadFrom": {[]string{"=io.Reader"}, []string{"int64", "error"}}, // io.ReaderFrom + "ReadRune": {[]string{}, []string{"rune", "int", "error"}}, // io.RuneReader + "Scan": {[]string{"=fmt.ScanState", "rune"}, []string{"error"}}, // fmt.Scanner + "Seek": {[]string{"=int64", "int"}, []string{"int64", "error"}}, // io.Seeker + "UnmarshalJSON": {[]string{"[]byte"}, []string{"error"}}, // json.Unmarshaler + "UnmarshalXML": {[]string{"*xml.Decoder", "xml.StartElement"}, []string{"error"}}, // xml.Unmarshaler + "UnreadByte": {[]string{}, []string{"error"}}, + "UnreadRune": {[]string{}, []string{"error"}}, + "WriteByte": {[]string{"byte"}, []string{"error"}}, // jpeg.writer (matching bufio.Writer) + "WriteTo": {[]string{"=io.Writer"}, []string{"int64", "error"}}, // io.WriterTo +} + +func checkCanonicalMethod(f *File, node ast.Node) { + switch n := node.(type) { + case *ast.FuncDecl: + if n.Recv != nil { + canonicalMethod(f, n.Name, n.Type) + } + case *ast.InterfaceType: + for _, field := range n.Methods.List { + for _, id := range field.Names { + canonicalMethod(f, id, field.Type.(*ast.FuncType)) + } + } + } +} + +func canonicalMethod(f *File, id *ast.Ident, t *ast.FuncType) { + // Expected input/output. + expect, ok := canonicalMethods[id.Name] + if !ok { + return + } + + // Actual input/output + args := typeFlatten(t.Params.List) + var results []ast.Expr + if t.Results != nil { + results = typeFlatten(t.Results.List) + } + + // Do the =s (if any) all match? + if !f.matchParams(expect.args, args, "=") || !f.matchParams(expect.results, results, "=") { + return + } + + // Everything must match. + if !f.matchParams(expect.args, args, "") || !f.matchParams(expect.results, results, "") { + expectFmt := id.Name + "(" + argjoin(expect.args) + ")" + if len(expect.results) == 1 { + expectFmt += " " + argjoin(expect.results) + } else if len(expect.results) > 1 { + expectFmt += " (" + argjoin(expect.results) + ")" + } + + f.b.Reset() + if err := printer.Fprint(&f.b, f.fset, t); err != nil { + fmt.Fprintf(&f.b, "<%s>", err) + } + actual := f.b.String() + actual = strings.TrimPrefix(actual, "func") + actual = id.Name + actual + + f.Badf(id.Pos(), "method %s should have signature %s", actual, expectFmt) + } +} + +func argjoin(x []string) string { + y := make([]string, len(x)) + for i, s := range x { + if s[0] == '=' { + s = s[1:] + } + y[i] = s + } + return strings.Join(y, ", ") +} + +// Turn parameter list into slice of types +// (in the ast, types are Exprs). +// Have to handle f(int, bool) and f(x, y, z int) +// so not a simple 1-to-1 conversion. +func typeFlatten(l []*ast.Field) []ast.Expr { + var t []ast.Expr + for _, f := range l { + if len(f.Names) == 0 { + t = append(t, f.Type) + continue + } + for _ = range f.Names { + t = append(t, f.Type) + } + } + return t +} + +// Does each type in expect with the given prefix match the corresponding type in actual? +func (f *File) matchParams(expect []string, actual []ast.Expr, prefix string) bool { + for i, x := range expect { + if !strings.HasPrefix(x, prefix) { + continue + } + if i >= len(actual) { + return false + } + if !f.matchParamType(x, actual[i]) { + return false + } + } + if prefix == "" && len(actual) > len(expect) { + return false + } + return true +} + +// Does this one type match? +func (f *File) matchParamType(expect string, actual ast.Expr) bool { + if strings.HasPrefix(expect, "=") { + expect = expect[1:] + } + // Strip package name if we're in that package. + if n := len(f.file.Name.Name); len(expect) > n && expect[:n] == f.file.Name.Name && expect[n] == '.' { + expect = expect[n+1:] + } + + // Overkill but easy. + f.b.Reset() + printer.Fprint(&f.b, f.fset, actual) + return f.b.String() == expect +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/nilfunc.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/nilfunc.go new file mode 100644 index 000000000..bfe05e335 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/nilfunc.go @@ -0,0 +1,67 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +This file contains the code to check for useless function comparisons. +A useless comparison is one like f == nil as opposed to f() == nil. +*/ + +package main + +import ( + "go/ast" + "go/token" + "go/types" +) + +func init() { + register("nilfunc", + "check for comparisons between functions and nil", + checkNilFuncComparison, + binaryExpr) +} + +func checkNilFuncComparison(f *File, node ast.Node) { + e := node.(*ast.BinaryExpr) + + // Only want == or != comparisons. + if e.Op != token.EQL && e.Op != token.NEQ { + return + } + + // Only want comparisons with a nil identifier on one side. + var e2 ast.Expr + switch { + case f.isNil(e.X): + e2 = e.Y + case f.isNil(e.Y): + e2 = e.X + default: + return + } + + // Only want identifiers or selector expressions. + var obj types.Object + switch v := e2.(type) { + case *ast.Ident: + obj = f.pkg.uses[v] + case *ast.SelectorExpr: + obj = f.pkg.uses[v.Sel] + default: + return + } + + // Only want functions. + if _, ok := obj.(*types.Func); !ok { + return + } + + f.Badf(e.Pos(), "comparison of function %v %v nil is always %v", obj.Name(), e.Op, e.Op == token.NEQ) +} + +// isNil reports whether the provided expression is the built-in nil +// identifier. +func (f *File) isNil(e ast.Expr) bool { + return f.pkg.types[e].Type == types.Typ[types.UntypedNil] +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/print.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/print.go new file mode 100644 index 000000000..f4b985cfb --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/print.go @@ -0,0 +1,650 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the printf-checker. + +package main + +import ( + "bytes" + "flag" + "go/ast" + "go/constant" + "go/token" + "go/types" + "strconv" + "strings" + "unicode/utf8" +) + +var printfuncs = flag.String("printfuncs", "", "comma-separated list of print function names to check") + +func init() { + register("printf", + "check printf-like invocations", + checkFmtPrintfCall, + funcDecl, callExpr) +} + +func initPrintFlags() { + if *printfuncs == "" { + return + } + for _, name := range strings.Split(*printfuncs, ",") { + if len(name) == 0 { + flag.Usage() + } + + // Backwards compatibility: skip optional first argument + // index after the colon. + if colon := strings.LastIndex(name, ":"); colon > 0 { + name = name[:colon] + } + + name = strings.ToLower(name) + if name[len(name)-1] == 'f' { + isFormattedPrint[name] = true + } else { + isPrint[name] = true + } + } +} + +// isFormattedPrint records the formatted-print functions. Names are +// lower-cased so the lookup is case insensitive. +var isFormattedPrint = map[string]bool{ + "errorf": true, + "fatalf": true, + "fprintf": true, + "logf": true, + "panicf": true, + "printf": true, + "sprintf": true, +} + +// isPrint records the unformatted-print functions. Names are lower-cased +// so the lookup is case insensitive. +var isPrint = map[string]bool{ + "error": true, + "fatal": true, + "fprint": true, + "fprintln": true, + "log": true, + "panic": true, + "panicln": true, + "print": true, + "println": true, + "sprint": true, + "sprintln": true, +} + +// formatString returns the format string argument and its index within +// the given printf-like call expression. +// +// The last parameter before variadic arguments is assumed to be +// a format string. +// +// The first string literal or string constant is assumed to be a format string +// if the call's signature cannot be determined. +// +// If it cannot find any format string parameter, it returns ("", -1). +func formatString(f *File, call *ast.CallExpr) (string, int) { + typ := f.pkg.types[call.Fun].Type + if typ != nil { + if sig, ok := typ.(*types.Signature); ok { + if !sig.Variadic() { + // Skip checking non-variadic functions + return "", -1 + } + idx := sig.Params().Len() - 2 + if idx < 0 { + // Skip checking variadic functions without + // fixed arguments. + return "", -1 + } + s, ok := stringLiteralArg(f, call, idx) + if !ok { + // The last argument before variadic args isn't a string + return "", -1 + } + return s, idx + } + } + + // Cannot determine call's signature. Fallback to scanning for the first + // string argument in the call + for idx := range call.Args { + if s, ok := stringLiteralArg(f, call, idx); ok { + return s, idx + } + } + return "", -1 +} + +// stringLiteralArg returns call's string constant argument at the index idx. +// +// ("", false) is returned if call's argument at the index idx isn't a string +// literal. +func stringLiteralArg(f *File, call *ast.CallExpr, idx int) (string, bool) { + if idx >= len(call.Args) { + return "", false + } + arg := call.Args[idx] + lit := f.pkg.types[arg].Value + if lit != nil && lit.Kind() == constant.String { + return constant.StringVal(lit), true + } + return "", false +} + +// checkCall triggers the print-specific checks if the call invokes a print function. +func checkFmtPrintfCall(f *File, node ast.Node) { + if d, ok := node.(*ast.FuncDecl); ok && isStringer(f, d) { + // Remember we saw this. + if f.stringers == nil { + f.stringers = make(map[*ast.Object]bool) + } + if l := d.Recv.List; len(l) == 1 { + if n := l[0].Names; len(n) == 1 { + f.stringers[n[0].Obj] = true + } + } + return + } + + call, ok := node.(*ast.CallExpr) + if !ok { + return + } + var Name string + switch x := call.Fun.(type) { + case *ast.Ident: + Name = x.Name + case *ast.SelectorExpr: + Name = x.Sel.Name + default: + return + } + + name := strings.ToLower(Name) + if _, ok := isFormattedPrint[name]; ok { + f.checkPrintf(call, Name) + return + } + if _, ok := isPrint[name]; ok { + f.checkPrint(call, Name) + return + } +} + +// isStringer returns true if the provided declaration is a "String() string" +// method, an implementation of fmt.Stringer. +func isStringer(f *File, d *ast.FuncDecl) bool { + return d.Recv != nil && d.Name.Name == "String" && d.Type.Results != nil && + len(d.Type.Params.List) == 0 && len(d.Type.Results.List) == 1 && + f.pkg.types[d.Type.Results.List[0].Type].Type == types.Typ[types.String] +} + +// formatState holds the parsed representation of a printf directive such as "%3.*[4]d". +// It is constructed by parsePrintfVerb. +type formatState struct { + verb rune // the format verb: 'd' for "%d" + format string // the full format directive from % through verb, "%.3d". + name string // Printf, Sprintf etc. + flags []byte // the list of # + etc. + argNums []int // the successive argument numbers that are consumed, adjusted to refer to actual arg in call + indexed bool // whether an indexing expression appears: %[1]d. + firstArg int // Index of first argument after the format in the Printf call. + // Used only during parse. + file *File + call *ast.CallExpr + argNum int // Which argument we're expecting to format now. + indexPending bool // Whether we have an indexed argument that has not resolved. + nbytes int // number of bytes of the format string consumed. +} + +// checkPrintf checks a call to a formatted print routine such as Printf. +func (f *File) checkPrintf(call *ast.CallExpr, name string) { + format, idx := formatString(f, call) + if idx < 0 { + if *verbose { + f.Warn(call.Pos(), "can't check non-constant format in call to", name) + } + return + } + + firstArg := idx + 1 // Arguments are immediately after format string. + if !strings.Contains(format, "%") { + if len(call.Args) > firstArg { + f.Badf(call.Pos(), "no formatting directive in %s call", name) + } + return + } + // Hard part: check formats against args. + argNum := firstArg + indexed := false + for i, w := 0, 0; i < len(format); i += w { + w = 1 + if format[i] == '%' { + state := f.parsePrintfVerb(call, name, format[i:], firstArg, argNum) + if state == nil { + return + } + w = len(state.format) + if state.indexed { + indexed = true + } + if !f.okPrintfArg(call, state) { // One error per format is enough. + return + } + if len(state.argNums) > 0 { + // Continue with the next sequential argument. + argNum = state.argNums[len(state.argNums)-1] + 1 + } + } + } + // Dotdotdot is hard. + if call.Ellipsis.IsValid() && argNum >= len(call.Args)-1 { + return + } + // If the arguments were direct indexed, we assume the programmer knows what's up. + // Otherwise, there should be no leftover arguments. + if !indexed && argNum != len(call.Args) { + expect := argNum - firstArg + numArgs := len(call.Args) - firstArg + f.Badf(call.Pos(), "wrong number of args for format in %s call: %d needed but %d args", name, expect, numArgs) + } +} + +// parseFlags accepts any printf flags. +func (s *formatState) parseFlags() { + for s.nbytes < len(s.format) { + switch c := s.format[s.nbytes]; c { + case '#', '0', '+', '-', ' ': + s.flags = append(s.flags, c) + s.nbytes++ + default: + return + } + } +} + +// scanNum advances through a decimal number if present. +func (s *formatState) scanNum() { + for ; s.nbytes < len(s.format); s.nbytes++ { + c := s.format[s.nbytes] + if c < '0' || '9' < c { + return + } + } +} + +// parseIndex scans an index expression. It returns false if there is a syntax error. +func (s *formatState) parseIndex() bool { + if s.nbytes == len(s.format) || s.format[s.nbytes] != '[' { + return true + } + // Argument index present. + s.indexed = true + s.nbytes++ // skip '[' + start := s.nbytes + s.scanNum() + if s.nbytes == len(s.format) || s.nbytes == start || s.format[s.nbytes] != ']' { + s.file.Badf(s.call.Pos(), "illegal syntax for printf argument index") + return false + } + arg32, err := strconv.ParseInt(s.format[start:s.nbytes], 10, 32) + if err != nil { + s.file.Badf(s.call.Pos(), "illegal syntax for printf argument index: %s", err) + return false + } + s.nbytes++ // skip ']' + arg := int(arg32) + arg += s.firstArg - 1 // We want to zero-index the actual arguments. + s.argNum = arg + s.indexPending = true + return true +} + +// parseNum scans a width or precision (or *). It returns false if there's a bad index expression. +func (s *formatState) parseNum() bool { + if s.nbytes < len(s.format) && s.format[s.nbytes] == '*' { + if s.indexPending { // Absorb it. + s.indexPending = false + } + s.nbytes++ + s.argNums = append(s.argNums, s.argNum) + s.argNum++ + } else { + s.scanNum() + } + return true +} + +// parsePrecision scans for a precision. It returns false if there's a bad index expression. +func (s *formatState) parsePrecision() bool { + // If there's a period, there may be a precision. + if s.nbytes < len(s.format) && s.format[s.nbytes] == '.' { + s.flags = append(s.flags, '.') // Treat precision as a flag. + s.nbytes++ + if !s.parseIndex() { + return false + } + if !s.parseNum() { + return false + } + } + return true +} + +// parsePrintfVerb looks the formatting directive that begins the format string +// and returns a formatState that encodes what the directive wants, without looking +// at the actual arguments present in the call. The result is nil if there is an error. +func (f *File) parsePrintfVerb(call *ast.CallExpr, name, format string, firstArg, argNum int) *formatState { + state := &formatState{ + format: format, + name: name, + flags: make([]byte, 0, 5), + argNum: argNum, + argNums: make([]int, 0, 1), + nbytes: 1, // There's guaranteed to be a percent sign. + indexed: false, + firstArg: firstArg, + file: f, + call: call, + } + // There may be flags. + state.parseFlags() + indexPending := false + // There may be an index. + if !state.parseIndex() { + return nil + } + // There may be a width. + if !state.parseNum() { + return nil + } + // There may be a precision. + if !state.parsePrecision() { + return nil + } + // Now a verb, possibly prefixed by an index (which we may already have). + if !indexPending && !state.parseIndex() { + return nil + } + if state.nbytes == len(state.format) { + f.Badf(call.Pos(), "missing verb at end of format string in %s call", name) + return nil + } + verb, w := utf8.DecodeRuneInString(state.format[state.nbytes:]) + state.verb = verb + state.nbytes += w + if verb != '%' { + state.argNums = append(state.argNums, state.argNum) + } + state.format = state.format[:state.nbytes] + return state +} + +// printfArgType encodes the types of expressions a printf verb accepts. It is a bitmask. +type printfArgType int + +const ( + argBool printfArgType = 1 << iota + argInt + argRune + argString + argFloat + argComplex + argPointer + anyType printfArgType = ^0 +) + +type printVerb struct { + verb rune // User may provide verb through Formatter; could be a rune. + flags string // known flags are all ASCII + typ printfArgType +} + +// Common flag sets for printf verbs. +const ( + noFlag = "" + numFlag = " -+.0" + sharpNumFlag = " -+.0#" + allFlags = " -+.0#" +) + +// printVerbs identifies which flags are known to printf for each verb. +// TODO: A type that implements Formatter may do what it wants, and vet +// will complain incorrectly. +var printVerbs = []printVerb{ + // '-' is a width modifier, always valid. + // '.' is a precision for float, max width for strings. + // '+' is required sign for numbers, Go format for %v. + // '#' is alternate format for several verbs. + // ' ' is spacer for numbers + {'%', noFlag, 0}, + {'b', numFlag, argInt | argFloat | argComplex}, + {'c', "-", argRune | argInt}, + {'d', numFlag, argInt}, + {'e', numFlag, argFloat | argComplex}, + {'E', numFlag, argFloat | argComplex}, + {'f', numFlag, argFloat | argComplex}, + {'F', numFlag, argFloat | argComplex}, + {'g', numFlag, argFloat | argComplex}, + {'G', numFlag, argFloat | argComplex}, + {'o', sharpNumFlag, argInt}, + {'p', "-#", argPointer}, + {'q', " -+.0#", argRune | argInt | argString}, + {'s', " -+.0", argString}, + {'t', "-", argBool}, + {'T', "-", anyType}, + {'U', "-#", argRune | argInt}, + {'v', allFlags, anyType}, + {'x', sharpNumFlag, argRune | argInt | argString}, + {'X', sharpNumFlag, argRune | argInt | argString}, +} + +// okPrintfArg compares the formatState to the arguments actually present, +// reporting any discrepancies it can discern. If the final argument is ellipsissed, +// there's little it can do for that. +func (f *File) okPrintfArg(call *ast.CallExpr, state *formatState) (ok bool) { + var v printVerb + found := false + // Linear scan is fast enough for a small list. + for _, v = range printVerbs { + if v.verb == state.verb { + found = true + break + } + } + if !found { + f.Badf(call.Pos(), "unrecognized printf verb %q", state.verb) + return false + } + for _, flag := range state.flags { + if !strings.ContainsRune(v.flags, rune(flag)) { + f.Badf(call.Pos(), "unrecognized printf flag for verb %q: %q", state.verb, flag) + return false + } + } + // Verb is good. If len(state.argNums)>trueArgs, we have something like %.*s and all + // but the final arg must be an integer. + trueArgs := 1 + if state.verb == '%' { + trueArgs = 0 + } + nargs := len(state.argNums) + for i := 0; i < nargs-trueArgs; i++ { + argNum := state.argNums[i] + if !f.argCanBeChecked(call, i, true, state) { + return + } + arg := call.Args[argNum] + if !f.matchArgType(argInt, nil, arg) { + f.Badf(call.Pos(), "arg %s for * in printf format not of type int", f.gofmt(arg)) + return false + } + } + if state.verb == '%' { + return true + } + argNum := state.argNums[len(state.argNums)-1] + if !f.argCanBeChecked(call, len(state.argNums)-1, false, state) { + return false + } + arg := call.Args[argNum] + if f.isFunctionValue(arg) && state.verb != 'p' && state.verb != 'T' { + f.Badf(call.Pos(), "arg %s in printf call is a function value, not a function call", f.gofmt(arg)) + return false + } + if !f.matchArgType(v.typ, nil, arg) { + typeString := "" + if typ := f.pkg.types[arg].Type; typ != nil { + typeString = typ.String() + } + f.Badf(call.Pos(), "arg %s for printf verb %%%c of wrong type: %s", f.gofmt(arg), state.verb, typeString) + return false + } + if v.typ&argString != 0 && v.verb != 'T' && !bytes.Contains(state.flags, []byte{'#'}) && f.recursiveStringer(arg) { + f.Badf(call.Pos(), "arg %s for printf causes recursive call to String method", f.gofmt(arg)) + return false + } + return true +} + +// recursiveStringer reports whether the provided argument is r or &r for the +// fmt.Stringer receiver identifier r. +func (f *File) recursiveStringer(e ast.Expr) bool { + if len(f.stringers) == 0 { + return false + } + var obj *ast.Object + switch e := e.(type) { + case *ast.Ident: + obj = e.Obj + case *ast.UnaryExpr: + if id, ok := e.X.(*ast.Ident); ok && e.Op == token.AND { + obj = id.Obj + } + } + + // It's unlikely to be a recursive stringer if it has a Format method. + if typ := f.pkg.types[e].Type; typ != nil { + // Not a perfect match; see issue 6259. + if f.hasMethod(typ, "Format") { + return false + } + } + + // We compare the underlying Object, which checks that the identifier + // is the one we declared as the receiver for the String method in + // which this printf appears. + return f.stringers[obj] +} + +// isFunctionValue reports whether the expression is a function as opposed to a function call. +// It is almost always a mistake to print a function value. +func (f *File) isFunctionValue(e ast.Expr) bool { + if typ := f.pkg.types[e].Type; typ != nil { + _, ok := typ.(*types.Signature) + return ok + } + return false +} + +// argCanBeChecked reports whether the specified argument is statically present; +// it may be beyond the list of arguments or in a terminal slice... argument, which +// means we can't see it. +func (f *File) argCanBeChecked(call *ast.CallExpr, formatArg int, isStar bool, state *formatState) bool { + argNum := state.argNums[formatArg] + if argNum < 0 { + // Shouldn't happen, so catch it with prejudice. + panic("negative arg num") + } + if argNum == 0 { + f.Badf(call.Pos(), `index value [0] for %s("%s"); indexes start at 1`, state.name, state.format) + return false + } + if argNum < len(call.Args)-1 { + return true // Always OK. + } + if call.Ellipsis.IsValid() { + return false // We just can't tell; there could be many more arguments. + } + if argNum < len(call.Args) { + return true + } + // There are bad indexes in the format or there are fewer arguments than the format needs. + // This is the argument number relative to the format: Printf("%s", "hi") will give 1 for the "hi". + arg := argNum - state.firstArg + 1 // People think of arguments as 1-indexed. + f.Badf(call.Pos(), `missing argument for %s("%s"): format reads arg %d, have only %d args`, state.name, state.format, arg, len(call.Args)-state.firstArg) + return false +} + +// checkPrint checks a call to an unformatted print routine such as Println. +func (f *File) checkPrint(call *ast.CallExpr, name string) { + firstArg := 0 + typ := f.pkg.types[call.Fun].Type + if typ == nil { + // Skip checking functions with unknown type. + return + } + if sig, ok := typ.(*types.Signature); ok { + if !sig.Variadic() { + // Skip checking non-variadic functions. + return + } + params := sig.Params() + firstArg = params.Len() - 1 + + typ := params.At(firstArg).Type() + typ = typ.(*types.Slice).Elem() + it, ok := typ.(*types.Interface) + if !ok || !it.Empty() { + // Skip variadic functions accepting non-interface{} args. + return + } + } + args := call.Args + if len(args) <= firstArg { + // Skip calls without variadic args. + return + } + args = args[firstArg:] + + // check for Println(os.Stderr, ...) + if firstArg == 0 { + if sel, ok := args[0].(*ast.SelectorExpr); ok { + if x, ok := sel.X.(*ast.Ident); ok { + if x.Name == "os" && strings.HasPrefix(sel.Sel.Name, "Std") { + f.Badf(call.Pos(), "first argument to %s is %s.%s", name, x.Name, sel.Sel.Name) + } + } + } + } + arg := args[0] + if lit, ok := arg.(*ast.BasicLit); ok && lit.Kind == token.STRING { + if strings.Contains(lit.Value, "%") { + f.Badf(call.Pos(), "possible formatting directive in %s call", name) + } + } + if strings.HasSuffix(name, "ln") { + // The last item, if a string, should not have a newline. + arg = args[len(args)-1] + if lit, ok := arg.(*ast.BasicLit); ok && lit.Kind == token.STRING { + if strings.HasSuffix(lit.Value, `\n"`) { + f.Badf(call.Pos(), "%s call ends with newline", name) + } + } + } + for _, arg := range args { + if f.isFunctionValue(arg) { + f.Badf(call.Pos(), "arg %s in %s call is a function value, not a function call", f.gofmt(arg), name) + } + if f.recursiveStringer(arg) { + f.Badf(call.Pos(), "arg %s in %s call causes recursive call to String method", f.gofmt(arg), name) + } + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/rangeloop.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/rangeloop.go new file mode 100644 index 000000000..e085e21a2 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/rangeloop.go @@ -0,0 +1,74 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +This file contains the code to check range loop variables bound inside function +literals that are deferred or launched in new goroutines. We only check +instances where the defer or go statement is the last statement in the loop +body, as otherwise we would need whole program analysis. + +For example: + + for i, v := range s { + go func() { + println(i, v) // not what you might expect + }() + } + +See: https://golang.org/doc/go_faq.html#closures_and_goroutines +*/ + +package main + +import "go/ast" + +func init() { + register("rangeloops", + "check that range loop variables are used correctly", + checkRangeLoop, + rangeStmt) +} + +// checkRangeLoop walks the body of the provided range statement, checking if +// its index or value variables are used unsafely inside goroutines or deferred +// function literals. +func checkRangeLoop(f *File, node ast.Node) { + n := node.(*ast.RangeStmt) + key, _ := n.Key.(*ast.Ident) + val, _ := n.Value.(*ast.Ident) + if key == nil && val == nil { + return + } + sl := n.Body.List + if len(sl) == 0 { + return + } + var last *ast.CallExpr + switch s := sl[len(sl)-1].(type) { + case *ast.GoStmt: + last = s.Call + case *ast.DeferStmt: + last = s.Call + default: + return + } + lit, ok := last.Fun.(*ast.FuncLit) + if !ok { + return + } + ast.Inspect(lit.Body, func(n ast.Node) bool { + id, ok := n.(*ast.Ident) + if !ok || id.Obj == nil { + return true + } + if f.pkg.types[id].Type == nil { + // Not referring to a variable + return true + } + if key != nil && id.Obj == key.Obj || val != nil && id.Obj == val.Obj { + f.Bad(id.Pos(), "range variable", id.Name, "captured by func literal") + } + return true + }) +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/shadow.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/shadow.go new file mode 100644 index 000000000..29c952fd8 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/shadow.go @@ -0,0 +1,246 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +This file contains the code to check for shadowed variables. +A shadowed variable is a variable declared in an inner scope +with the same name and type as a variable in an outer scope, +and where the outer variable is mentioned after the inner one +is declared. + +(This definition can be refined; the module generates too many +false positives and is not yet enabled by default.) + +For example: + + func BadRead(f *os.File, buf []byte) error { + var err error + for { + n, err := f.Read(buf) // shadows the function variable 'err' + if err != nil { + break // causes return of wrong value + } + foo(buf) + } + return err + } + +*/ + +package main + +import ( + "flag" + "go/ast" + "go/token" + "go/types" +) + +var strictShadowing = flag.Bool("shadowstrict", false, "whether to be strict about shadowing; can be noisy") + +func init() { + register("shadow", + "check for shadowed variables (experimental; must be set explicitly)", + checkShadow, + assignStmt, genDecl) + experimental["shadow"] = true +} + +func checkShadow(f *File, node ast.Node) { + switch n := node.(type) { + case *ast.AssignStmt: + checkShadowAssignment(f, n) + case *ast.GenDecl: + checkShadowDecl(f, n) + } +} + +// Span stores the minimum range of byte positions in the file in which a +// given variable (types.Object) is mentioned. It is lexically defined: it spans +// from the beginning of its first mention to the end of its last mention. +// A variable is considered shadowed (if *strictShadowing is off) only if the +// shadowing variable is declared within the span of the shadowed variable. +// In other words, if a variable is shadowed but not used after the shadowed +// variable is declared, it is inconsequential and not worth complaining about. +// This simple check dramatically reduces the nuisance rate for the shadowing +// check, at least until something cleverer comes along. +// +// One wrinkle: A "naked return" is a silent use of a variable that the Span +// will not capture, but the compilers catch naked returns of shadowed +// variables so we don't need to. +// +// Cases this gets wrong (TODO): +// - If a for loop's continuation statement mentions a variable redeclared in +// the block, we should complain about it but don't. +// - A variable declared inside a function literal can falsely be identified +// as shadowing a variable in the outer function. +// +type Span struct { + min token.Pos + max token.Pos +} + +// contains reports whether the position is inside the span. +func (s Span) contains(pos token.Pos) bool { + return s.min <= pos && pos < s.max +} + +// growSpan expands the span for the object to contain the instance represented +// by the identifier. +func (pkg *Package) growSpan(ident *ast.Ident, obj types.Object) { + if *strictShadowing { + return // No need + } + pos := ident.Pos() + end := ident.End() + span, ok := pkg.spans[obj] + if ok { + if span.min > pos { + span.min = pos + } + if span.max < end { + span.max = end + } + } else { + span = Span{pos, end} + } + pkg.spans[obj] = span +} + +// checkShadowAssignment checks for shadowing in a short variable declaration. +func checkShadowAssignment(f *File, a *ast.AssignStmt) { + if a.Tok != token.DEFINE { + return + } + if f.idiomaticShortRedecl(a) { + return + } + for _, expr := range a.Lhs { + ident, ok := expr.(*ast.Ident) + if !ok { + f.Badf(expr.Pos(), "invalid AST: short variable declaration of non-identifier") + return + } + checkShadowing(f, ident) + } +} + +// idiomaticShortRedecl reports whether this short declaration can be ignored for +// the purposes of shadowing, that is, that any redeclarations it contains are deliberate. +func (f *File) idiomaticShortRedecl(a *ast.AssignStmt) bool { + // Don't complain about deliberate redeclarations of the form + // i := i + // Such constructs are idiomatic in range loops to create a new variable + // for each iteration. Another example is + // switch n := n.(type) + if len(a.Rhs) != len(a.Lhs) { + return false + } + // We know it's an assignment, so the LHS must be all identifiers. (We check anyway.) + for i, expr := range a.Lhs { + lhs, ok := expr.(*ast.Ident) + if !ok { + f.Badf(expr.Pos(), "invalid AST: short variable declaration of non-identifier") + return true // Don't do any more processing. + } + switch rhs := a.Rhs[i].(type) { + case *ast.Ident: + if lhs.Name != rhs.Name { + return false + } + case *ast.TypeAssertExpr: + if id, ok := rhs.X.(*ast.Ident); ok { + if lhs.Name != id.Name { + return false + } + } + default: + return false + } + } + return true +} + +// idiomaticRedecl reports whether this declaration spec can be ignored for +// the purposes of shadowing, that is, that any redeclarations it contains are deliberate. +func (f *File) idiomaticRedecl(d *ast.ValueSpec) bool { + // Don't complain about deliberate redeclarations of the form + // var i, j = i, j + if len(d.Names) != len(d.Values) { + return false + } + for i, lhs := range d.Names { + if rhs, ok := d.Values[i].(*ast.Ident); ok { + if lhs.Name != rhs.Name { + return false + } + } + } + return true +} + +// checkShadowDecl checks for shadowing in a general variable declaration. +func checkShadowDecl(f *File, d *ast.GenDecl) { + if d.Tok != token.VAR { + return + } + for _, spec := range d.Specs { + valueSpec, ok := spec.(*ast.ValueSpec) + if !ok { + f.Badf(spec.Pos(), "invalid AST: var GenDecl not ValueSpec") + return + } + // Don't complain about deliberate redeclarations of the form + // var i = i + if f.idiomaticRedecl(valueSpec) { + return + } + for _, ident := range valueSpec.Names { + checkShadowing(f, ident) + } + } +} + +// checkShadowing checks whether the identifier shadows an identifier in an outer scope. +func checkShadowing(f *File, ident *ast.Ident) { + if ident.Name == "_" { + // Can't shadow the blank identifier. + return + } + obj := f.pkg.defs[ident] + if obj == nil { + return + } + // obj.Parent.Parent is the surrounding scope. If we can find another declaration + // starting from there, we have a shadowed identifier. + _, shadowed := obj.Parent().Parent().LookupParent(obj.Name(), obj.Pos()) + if shadowed == nil { + return + } + // Don't complain if it's shadowing a universe-declared identifier; that's fine. + if shadowed.Parent() == types.Universe { + return + } + if *strictShadowing { + // The shadowed identifier must appear before this one to be an instance of shadowing. + if shadowed.Pos() > ident.Pos() { + return + } + } else { + // Don't complain if the span of validity of the shadowed identifier doesn't include + // the shadowing identifier. + span, ok := f.pkg.spans[shadowed] + if !ok { + f.Badf(ident.Pos(), "internal error: no range for %q", ident.Name) + return + } + if !span.contains(ident.Pos()) { + return + } + } + // Don't complain if the types differ: that implies the programmer really wants two different things. + if types.Identical(obj.Type(), shadowed.Type()) { + f.Badf(ident.Pos(), "declaration of %q shadows declaration at %s", obj.Name(), f.loc(shadowed.Pos())) + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/shift.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/shift.go new file mode 100644 index 000000000..8c038b4bd --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/shift.go @@ -0,0 +1,82 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +This file contains the code to check for suspicious shifts. +*/ + +package main + +import ( + "go/ast" + "go/constant" + "go/token" + "go/types" +) + +func init() { + register("shift", + "check for useless shifts", + checkShift, + binaryExpr, assignStmt) +} + +func checkShift(f *File, node ast.Node) { + switch node := node.(type) { + case *ast.BinaryExpr: + if node.Op == token.SHL || node.Op == token.SHR { + checkLongShift(f, node, node.X, node.Y) + } + case *ast.AssignStmt: + if len(node.Lhs) != 1 || len(node.Rhs) != 1 { + return + } + if node.Tok == token.SHL_ASSIGN || node.Tok == token.SHR_ASSIGN { + checkLongShift(f, node, node.Lhs[0], node.Rhs[0]) + } + } +} + +// checkLongShift checks if shift or shift-assign operations shift by more than +// the length of the underlying variable. +func checkLongShift(f *File, node ast.Node, x, y ast.Expr) { + v := f.pkg.types[y].Value + if v == nil { + return + } + amt, ok := constant.Int64Val(v) + if !ok { + return + } + t := f.pkg.types[x].Type + if t == nil { + return + } + b, ok := t.Underlying().(*types.Basic) + if !ok { + return + } + var size int64 + var msg string + switch b.Kind() { + case types.Uint8, types.Int8: + size = 8 + case types.Uint16, types.Int16: + size = 16 + case types.Uint32, types.Int32: + size = 32 + case types.Uint64, types.Int64: + size = 64 + case types.Int, types.Uint, types.Uintptr: + // These types may be as small as 32 bits, but no smaller. + size = 32 + msg = "might be " + default: + return + } + if amt >= size { + ident := f.gofmt(x) + f.Badf(node.Pos(), "%s %stoo small for shift of %d", ident, msg, amt) + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/structtag.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/structtag.go new file mode 100644 index 000000000..abff14fb1 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/structtag.go @@ -0,0 +1,122 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the test for canonical struct tags. + +package main + +import ( + "errors" + "go/ast" + "reflect" + "strconv" +) + +func init() { + register("structtags", + "check that struct field tags have canonical format and apply to exported fields as needed", + checkCanonicalFieldTag, + field) +} + +// checkCanonicalFieldTag checks a struct field tag. +func checkCanonicalFieldTag(f *File, node ast.Node) { + field := node.(*ast.Field) + if field.Tag == nil { + return + } + + tag, err := strconv.Unquote(field.Tag.Value) + if err != nil { + f.Badf(field.Pos(), "unable to read struct tag %s", field.Tag.Value) + return + } + + if err := validateStructTag(tag); err != nil { + f.Badf(field.Pos(), "struct field tag %s not compatible with reflect.StructTag.Get: %s", field.Tag.Value, err) + } + + // Check for use of json or xml tags with unexported fields. + + // Embedded struct. Nothing to do for now, but that + // may change, depending on what happens with issue 7363. + if len(field.Names) == 0 { + return + } + + if field.Names[0].IsExported() { + return + } + + st := reflect.StructTag(tag) + for _, enc := range [...]string{"json", "xml"} { + if st.Get(enc) != "" { + f.Badf(field.Pos(), "struct field %s has %s tag but is not exported", field.Names[0].Name, enc) + return + } + } +} + +var ( + errTagSyntax = errors.New("bad syntax for struct tag pair") + errTagKeySyntax = errors.New("bad syntax for struct tag key") + errTagValueSyntax = errors.New("bad syntax for struct tag value") +) + +// validateStructTag parses the struct tag and returns an error if it is not +// in the canonical format, which is a space-separated list of key:"value" +// settings. The value may contain spaces. +func validateStructTag(tag string) error { + // This code is based on the StructTag.Get code in package reflect. + + for tag != "" { + // Skip leading space. + i := 0 + for i < len(tag) && tag[i] == ' ' { + i++ + } + tag = tag[i:] + if tag == "" { + break + } + + // Scan to colon. A space, a quote or a control character is a syntax error. + // Strictly speaking, control chars include the range [0x7f, 0x9f], not just + // [0x00, 0x1f], but in practice, we ignore the multi-byte control characters + // as it is simpler to inspect the tag's bytes than the tag's runes. + i = 0 + for i < len(tag) && tag[i] > ' ' && tag[i] != ':' && tag[i] != '"' && tag[i] != 0x7f { + i++ + } + if i == 0 { + return errTagKeySyntax + } + if i+1 >= len(tag) || tag[i] != ':' { + return errTagSyntax + } + if tag[i+1] != '"' { + return errTagValueSyntax + } + tag = tag[i+1:] + + // Scan quoted string to find value. + i = 1 + for i < len(tag) && tag[i] != '"' { + if tag[i] == '\\' { + i++ + } + i++ + } + if i >= len(tag) { + return errTagValueSyntax + } + qvalue := tag[:i+1] + tag = tag[i+1:] + + if _, err := strconv.Unquote(qvalue); err != nil { + return errTagValueSyntax + } + } + return nil +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/tests.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/tests.go new file mode 100644 index 000000000..8c051f133 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/tests.go @@ -0,0 +1,187 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" + "go/types" + "strings" + "unicode" + "unicode/utf8" +) + +func init() { + register("tests", + "check for common mistaken usages of tests/documentation examples", + checkTestFunctions, + funcDecl) +} + +func isExampleSuffix(s string) bool { + r, size := utf8.DecodeRuneInString(s) + return size > 0 && unicode.IsLower(r) +} + +func isTestSuffix(name string) bool { + if len(name) == 0 { + // "Test" is ok. + return true + } + r, _ := utf8.DecodeRuneInString(name) + return !unicode.IsLower(r) +} + +func isTestParam(typ ast.Expr, wantType string) bool { + ptr, ok := typ.(*ast.StarExpr) + if !ok { + // Not a pointer. + return false + } + // No easy way of making sure it's a *testing.T or *testing.B: + // ensure the name of the type matches. + if name, ok := ptr.X.(*ast.Ident); ok { + return name.Name == wantType + } + if sel, ok := ptr.X.(*ast.SelectorExpr); ok { + return sel.Sel.Name == wantType + } + return false +} + +func lookup(name string, scopes []*types.Scope) types.Object { + for _, scope := range scopes { + if o := scope.Lookup(name); o != nil { + return o + } + } + return nil +} + +func extendedScope(f *File) []*types.Scope { + scopes := []*types.Scope{f.pkg.typesPkg.Scope()} + if f.basePkg != nil { + scopes = append(scopes, f.basePkg.typesPkg.Scope()) + } else { + // If basePkg is not specified (e.g. when checking a single file) try to + // find it among imports. + pkgName := f.pkg.typesPkg.Name() + if strings.HasSuffix(pkgName, "_test") { + basePkgName := strings.TrimSuffix(pkgName, "_test") + for _, p := range f.pkg.typesPkg.Imports() { + if p.Name() == basePkgName { + scopes = append(scopes, p.Scope()) + break + } + } + } + } + return scopes +} + +func checkExample(fn *ast.FuncDecl, f *File, report reporter) { + fnName := fn.Name.Name + if params := fn.Type.Params; len(params.List) != 0 { + report("%s should be niladic", fnName) + } + if results := fn.Type.Results; results != nil && len(results.List) != 0 { + report("%s should return nothing", fnName) + } + + if filesRun && !includesNonTest { + // The coherence checks between a test and the package it tests + // will report false positives if no non-test files have + // been provided. + return + } + + if fnName == "Example" { + // Nothing more to do. + return + } + + var ( + exName = strings.TrimPrefix(fnName, "Example") + elems = strings.SplitN(exName, "_", 3) + ident = elems[0] + obj = lookup(ident, extendedScope(f)) + ) + if ident != "" && obj == nil { + // Check ExampleFoo and ExampleBadFoo. + report("%s refers to unknown identifier: %s", fnName, ident) + // Abort since obj is absent and no subsequent checks can be performed. + return + } + if len(elems) < 2 { + // Nothing more to do. + return + } + + if ident == "" { + // Check Example_suffix and Example_BadSuffix. + if residual := strings.TrimPrefix(exName, "_"); !isExampleSuffix(residual) { + report("%s has malformed example suffix: %s", fnName, residual) + } + return + } + + mmbr := elems[1] + if !isExampleSuffix(mmbr) { + // Check ExampleFoo_Method and ExampleFoo_BadMethod. + if obj, _, _ := types.LookupFieldOrMethod(obj.Type(), true, obj.Pkg(), mmbr); obj == nil { + report("%s refers to unknown field or method: %s.%s", fnName, ident, mmbr) + } + } + if len(elems) == 3 && !isExampleSuffix(elems[2]) { + // Check ExampleFoo_Method_suffix and ExampleFoo_Method_Badsuffix. + report("%s has malformed example suffix: %s", fnName, elems[2]) + } +} + +func checkTest(fn *ast.FuncDecl, prefix string, report reporter) { + // Want functions with 0 results and 1 parameter. + if fn.Type.Results != nil && len(fn.Type.Results.List) > 0 || + fn.Type.Params == nil || + len(fn.Type.Params.List) != 1 || + len(fn.Type.Params.List[0].Names) > 1 { + return + } + + // The param must look like a *testing.T or *testing.B. + if !isTestParam(fn.Type.Params.List[0].Type, prefix[:1]) { + return + } + + if !isTestSuffix(fn.Name.Name[len(prefix):]) { + report("%s has malformed name: first letter after '%s' must not be lowercase", fn.Name.Name, prefix) + } +} + +type reporter func(format string, args ...interface{}) + +// checkTestFunctions walks Test, Benchmark and Example functions checking +// malformed names, wrong signatures and examples documenting inexistent +// identifiers. +func checkTestFunctions(f *File, node ast.Node) { + if !strings.HasSuffix(f.name, "_test.go") { + return + } + + fn, ok := node.(*ast.FuncDecl) + if !ok || fn.Recv != nil { + // Ignore non-functions or functions with receivers. + return + } + + report := func(format string, args ...interface{}) { f.Badf(node.Pos(), format, args...) } + + switch { + case strings.HasPrefix(fn.Name.Name, "Example"): + checkExample(fn, f, report) + case strings.HasPrefix(fn.Name.Name, "Test"): + checkTest(fn, "Test", report) + case strings.HasPrefix(fn.Name.Name, "Benchmark"): + checkTest(fn, "Benchmark", report) + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/types.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/types.go new file mode 100644 index 000000000..4d0e6154b --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/types.go @@ -0,0 +1,281 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the pieces of the tool that use typechecking from the go/types package. + +package main + +import ( + "go/ast" + "go/importer" + "go/token" + "go/types" +) + +// stdImporter is the importer we use to import packages. +// It is created during initialization so that all packages +// are imported by the same importer. +var stdImporter = importer.Default() + +var ( + errorType *types.Interface + stringerType *types.Interface // possibly nil + formatterType *types.Interface // possibly nil +) + +func init() { + errorType = types.Universe.Lookup("error").Type().Underlying().(*types.Interface) + + if typ := importType("fmt", "Stringer"); typ != nil { + stringerType = typ.Underlying().(*types.Interface) + } + + if typ := importType("fmt", "Formatter"); typ != nil { + formatterType = typ.Underlying().(*types.Interface) + } +} + +// importType returns the type denoted by the qualified identifier +// path.name, and adds the respective package to the imports map +// as a side effect. In case of an error, importType returns nil. +func importType(path, name string) types.Type { + pkg, err := stdImporter.Import(path) + if err != nil { + // This can happen if the package at path hasn't been compiled yet. + warnf("import failed: %v", err) + return nil + } + if obj, ok := pkg.Scope().Lookup(name).(*types.TypeName); ok { + return obj.Type() + } + warnf("invalid type name %q", name) + return nil +} + +func (pkg *Package) check(fs *token.FileSet, astFiles []*ast.File) error { + pkg.defs = make(map[*ast.Ident]types.Object) + pkg.uses = make(map[*ast.Ident]types.Object) + pkg.selectors = make(map[*ast.SelectorExpr]*types.Selection) + pkg.spans = make(map[types.Object]Span) + pkg.types = make(map[ast.Expr]types.TypeAndValue) + config := types.Config{ + // We use the same importer for all imports to ensure that + // everybody sees identical packages for the given paths. + Importer: stdImporter, + // By providing a Config with our own error function, it will continue + // past the first error. There is no need for that function to do anything. + Error: func(error) {}, + } + info := &types.Info{ + Selections: pkg.selectors, + Types: pkg.types, + Defs: pkg.defs, + Uses: pkg.uses, + } + typesPkg, err := config.Check(pkg.path, fs, astFiles, info) + pkg.typesPkg = typesPkg + // update spans + for id, obj := range pkg.defs { + pkg.growSpan(id, obj) + } + for id, obj := range pkg.uses { + pkg.growSpan(id, obj) + } + return err +} + +// matchArgType reports an error if printf verb t is not appropriate +// for operand arg. +// +// typ is used only for recursive calls; external callers must supply nil. +// +// (Recursion arises from the compound types {map,chan,slice} which +// may be printed with %d etc. if that is appropriate for their element +// types.) +func (f *File) matchArgType(t printfArgType, typ types.Type, arg ast.Expr) bool { + return f.matchArgTypeInternal(t, typ, arg, make(map[types.Type]bool)) +} + +// matchArgTypeInternal is the internal version of matchArgType. It carries a map +// remembering what types are in progress so we don't recur when faced with recursive +// types or mutually recursive types. +func (f *File) matchArgTypeInternal(t printfArgType, typ types.Type, arg ast.Expr, inProgress map[types.Type]bool) bool { + // %v, %T accept any argument type. + if t == anyType { + return true + } + if typ == nil { + // external call + typ = f.pkg.types[arg].Type + if typ == nil { + return true // probably a type check problem + } + } + // If the type implements fmt.Formatter, we have nothing to check. + // formatterTyp may be nil - be conservative and check for Format method in that case. + if formatterType != nil && types.Implements(typ, formatterType) || f.hasMethod(typ, "Format") { + return true + } + // If we can use a string, might arg (dynamically) implement the Stringer or Error interface? + if t&argString != 0 { + if types.AssertableTo(errorType, typ) || stringerType != nil && types.AssertableTo(stringerType, typ) { + return true + } + } + + typ = typ.Underlying() + if inProgress[typ] { + // We're already looking at this type. The call that started it will take care of it. + return true + } + inProgress[typ] = true + + switch typ := typ.(type) { + case *types.Signature: + return t&argPointer != 0 + + case *types.Map: + // Recur: map[int]int matches %d. + return t&argPointer != 0 || + (f.matchArgTypeInternal(t, typ.Key(), arg, inProgress) && f.matchArgTypeInternal(t, typ.Elem(), arg, inProgress)) + + case *types.Chan: + return t&argPointer != 0 + + case *types.Array: + // Same as slice. + if types.Identical(typ.Elem().Underlying(), types.Typ[types.Byte]) && t&argString != 0 { + return true // %s matches []byte + } + // Recur: []int matches %d. + return t&argPointer != 0 || f.matchArgTypeInternal(t, typ.Elem().Underlying(), arg, inProgress) + + case *types.Slice: + // Same as array. + if types.Identical(typ.Elem().Underlying(), types.Typ[types.Byte]) && t&argString != 0 { + return true // %s matches []byte + } + // Recur: []int matches %d. But watch out for + // type T []T + // If the element is a pointer type (type T[]*T), it's handled fine by the Pointer case below. + return t&argPointer != 0 || f.matchArgTypeInternal(t, typ.Elem(), arg, inProgress) + + case *types.Pointer: + // Ugly, but dealing with an edge case: a known pointer to an invalid type, + // probably something from a failed import. + if typ.Elem().String() == "invalid type" { + if *verbose { + f.Warnf(arg.Pos(), "printf argument %v is pointer to invalid or unknown type", f.gofmt(arg)) + } + return true // special case + } + // If it's actually a pointer with %p, it prints as one. + if t == argPointer { + return true + } + // If it's pointer to struct, that's equivalent in our analysis to whether we can print the struct. + if str, ok := typ.Elem().Underlying().(*types.Struct); ok { + return f.matchStructArgType(t, str, arg, inProgress) + } + // The rest can print with %p as pointers, or as integers with %x etc. + return t&(argInt|argPointer) != 0 + + case *types.Struct: + return f.matchStructArgType(t, typ, arg, inProgress) + + case *types.Interface: + // If the static type of the argument is empty interface, there's little we can do. + // Example: + // func f(x interface{}) { fmt.Printf("%s", x) } + // Whether x is valid for %s depends on the type of the argument to f. One day + // we will be able to do better. For now, we assume that empty interface is OK + // but non-empty interfaces, with Stringer and Error handled above, are errors. + return typ.NumMethods() == 0 + + case *types.Basic: + switch typ.Kind() { + case types.UntypedBool, + types.Bool: + return t&argBool != 0 + + case types.UntypedInt, + types.Int, + types.Int8, + types.Int16, + types.Int32, + types.Int64, + types.Uint, + types.Uint8, + types.Uint16, + types.Uint32, + types.Uint64, + types.Uintptr: + return t&argInt != 0 + + case types.UntypedFloat, + types.Float32, + types.Float64: + return t&argFloat != 0 + + case types.UntypedComplex, + types.Complex64, + types.Complex128: + return t&argComplex != 0 + + case types.UntypedString, + types.String: + return t&argString != 0 + + case types.UnsafePointer: + return t&(argPointer|argInt) != 0 + + case types.UntypedRune: + return t&(argInt|argRune) != 0 + + case types.UntypedNil: + return t&argPointer != 0 // TODO? + + case types.Invalid: + if *verbose { + f.Warnf(arg.Pos(), "printf argument %v has invalid or unknown type", f.gofmt(arg)) + } + return true // Probably a type check problem. + } + panic("unreachable") + } + + return false +} + +// hasBasicType reports whether x's type is a types.Basic with the given kind. +func (f *File) hasBasicType(x ast.Expr, kind types.BasicKind) bool { + t := f.pkg.types[x].Type + if t != nil { + t = t.Underlying() + } + b, ok := t.(*types.Basic) + return ok && b.Kind() == kind +} + +// matchStructArgType reports whether all the elements of the struct match the expected +// type. For instance, with "%d" all the elements must be printable with the "%d" format. +func (f *File) matchStructArgType(t printfArgType, typ *types.Struct, arg ast.Expr, inProgress map[types.Type]bool) bool { + for i := 0; i < typ.NumFields(); i++ { + if !f.matchArgTypeInternal(t, typ.Field(i).Type(), arg, inProgress) { + return false + } + } + return true +} + +// hasMethod reports whether the type contains a method with the given name. +// It is part of the workaround for Formatters and should be deleted when +// that workaround is no longer necessary. +// TODO: This could be better once issue 6259 is fixed. +func (f *File) hasMethod(typ types.Type, name string) bool { + // assume we have an addressable variable of type typ + obj, _, _ := types.LookupFieldOrMethod(typ, true, f.pkg.typesPkg, name) + _, ok := obj.(*types.Func) + return ok +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/unsafeptr.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/unsafeptr.go new file mode 100644 index 000000000..a143e4d81 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/unsafeptr.go @@ -0,0 +1,97 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Check for invalid uintptr -> unsafe.Pointer conversions. + +package main + +import ( + "go/ast" + "go/token" + "go/types" +) + +func init() { + register("unsafeptr", + "check for misuse of unsafe.Pointer", + checkUnsafePointer, + callExpr) +} + +func checkUnsafePointer(f *File, node ast.Node) { + x := node.(*ast.CallExpr) + if len(x.Args) != 1 { + return + } + if f.hasBasicType(x.Fun, types.UnsafePointer) && f.hasBasicType(x.Args[0], types.Uintptr) && !f.isSafeUintptr(x.Args[0]) { + f.Badf(x.Pos(), "possible misuse of unsafe.Pointer") + } +} + +// isSafeUintptr reports whether x - already known to be a uintptr - +// is safe to convert to unsafe.Pointer. It is safe if x is itself derived +// directly from an unsafe.Pointer via conversion and pointer arithmetic +// or if x is the result of reflect.Value.Pointer or reflect.Value.UnsafeAddr +// or obtained from the Data field of a *reflect.SliceHeader or *reflect.StringHeader. +func (f *File) isSafeUintptr(x ast.Expr) bool { + switch x := x.(type) { + case *ast.ParenExpr: + return f.isSafeUintptr(x.X) + + case *ast.SelectorExpr: + switch x.Sel.Name { + case "Data": + // reflect.SliceHeader and reflect.StringHeader are okay, + // but only if they are pointing at a real slice or string. + // It's not okay to do: + // var x SliceHeader + // x.Data = uintptr(unsafe.Pointer(...)) + // ... use x ... + // p := unsafe.Pointer(x.Data) + // because in the middle the garbage collector doesn't + // see x.Data as a pointer and so x.Data may be dangling + // by the time we get to the conversion at the end. + // For now approximate by saying that *Header is okay + // but Header is not. + pt, ok := f.pkg.types[x.X].Type.(*types.Pointer) + if ok { + t, ok := pt.Elem().(*types.Named) + if ok && t.Obj().Pkg().Path() == "reflect" { + switch t.Obj().Name() { + case "StringHeader", "SliceHeader": + return true + } + } + } + } + + case *ast.CallExpr: + switch len(x.Args) { + case 0: + // maybe call to reflect.Value.Pointer or reflect.Value.UnsafeAddr. + sel, ok := x.Fun.(*ast.SelectorExpr) + if !ok { + break + } + switch sel.Sel.Name { + case "Pointer", "UnsafeAddr": + t, ok := f.pkg.types[sel.X].Type.(*types.Named) + if ok && t.Obj().Pkg().Path() == "reflect" && t.Obj().Name() == "Value" { + return true + } + } + + case 1: + // maybe conversion of uintptr to unsafe.Pointer + return f.hasBasicType(x.Fun, types.Uintptr) && f.hasBasicType(x.Args[0], types.UnsafePointer) + } + + case *ast.BinaryExpr: + switch x.Op { + case token.ADD, token.SUB: + return f.isSafeUintptr(x.X) && !f.isSafeUintptr(x.Y) + } + } + return false +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/unused.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/unused.go new file mode 100644 index 000000000..df2317a43 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/dnephin/govet/unused.go @@ -0,0 +1,93 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file defines the check for unused results of calls to certain +// pure functions. + +package main + +import ( + "flag" + "go/ast" + "go/token" + "go/types" + "strings" +) + +var unusedFuncsFlag = flag.String("unusedfuncs", + "errors.New,fmt.Errorf,fmt.Sprintf,fmt.Sprint,sort.Reverse", + "comma-separated list of functions whose results must be used") + +var unusedStringMethodsFlag = flag.String("unusedstringmethods", + "Error,String", + "comma-separated list of names of methods of type func() string whose results must be used") + +func init() { + register("unusedresult", + "check for unused result of calls to functions in -unusedfuncs list and methods in -unusedstringmethods list", + checkUnusedResult, + exprStmt) +} + +// func() string +var sigNoArgsStringResult = types.NewSignature(nil, nil, + types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.String])), + false) + +var unusedFuncs = make(map[string]bool) +var unusedStringMethods = make(map[string]bool) + +func initUnusedFlags() { + commaSplit := func(s string, m map[string]bool) { + if s != "" { + for _, name := range strings.Split(s, ",") { + if len(name) == 0 { + flag.Usage() + } + m[name] = true + } + } + } + commaSplit(*unusedFuncsFlag, unusedFuncs) + commaSplit(*unusedStringMethodsFlag, unusedStringMethods) +} + +func checkUnusedResult(f *File, n ast.Node) { + call, ok := unparen(n.(*ast.ExprStmt).X).(*ast.CallExpr) + if !ok { + return // not a call statement + } + fun := unparen(call.Fun) + + if f.pkg.types[fun].IsType() { + return // a conversion, not a call + } + + selector, ok := fun.(*ast.SelectorExpr) + if !ok { + return // neither a method call nor a qualified ident + } + + sel, ok := f.pkg.selectors[selector] + if ok && sel.Kind() == types.MethodVal { + // method (e.g. foo.String()) + obj := sel.Obj().(*types.Func) + sig := sel.Type().(*types.Signature) + if types.Identical(sig, sigNoArgsStringResult) { + if unusedStringMethods[obj.Name()] { + f.Badf(call.Lparen, "result of (%s).%s call not used", + sig.Recv().Type(), obj.Name()) + } + } + } else if !ok { + // package-qualified function (e.g. fmt.Errorf) + obj, _ := f.pkg.uses[selector.Sel] + if obj, ok := obj.(*types.Func); ok { + qname := obj.Pkg().Path() + "." + obj.Name() + if unusedFuncs[qname] { + f.Badf(call.Lparen, "result of %v call not used", qname) + } + } + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/go16.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/go16-18.go similarity index 90% rename from vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/go16.go rename to vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/go16-18.go index 6a70285f1..f25cec14a 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/go16.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/go16-18.go @@ -1,4 +1,4 @@ -// +build go1.6 +// +build go1.6,!go1.9 package gotool diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/internal/load/path.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/internal/load/path.go new file mode 100644 index 000000000..74e15b9d3 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/internal/load/path.go @@ -0,0 +1,27 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 + +package load + +import ( + "strings" +) + +// hasPathPrefix reports whether the path s begins with the +// elements in prefix. +func hasPathPrefix(s, prefix string) bool { + switch { + default: + return false + case len(s) == len(prefix): + return s == prefix + case len(s) > len(prefix): + if prefix != "" && prefix[len(prefix)-1] == '/' { + return strings.HasPrefix(s, prefix) + } + return s[len(prefix)] == '/' && s[:len(prefix)] == prefix + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/internal/load/pkg.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/internal/load/pkg.go new file mode 100644 index 000000000..b937ede75 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/internal/load/pkg.go @@ -0,0 +1,25 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 + +// Package load loads packages. +package load + +import ( + "strings" +) + +// isStandardImportPath reports whether $GOROOT/src/path should be considered +// part of the standard distribution. For historical reasons we allow people to add +// their own code to $GOROOT instead of using $GOPATH, but we assume that +// code will start with a domain name (dot in the first element). +func isStandardImportPath(path string) bool { + i := strings.Index(path, "/") + if i < 0 { + i = len(path) + } + elem := path[:i] + return !strings.Contains(elem, ".") +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/internal/load/search.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/internal/load/search.go new file mode 100644 index 000000000..17ed62dda --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/internal/load/search.go @@ -0,0 +1,354 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 + +package load + +import ( + "fmt" + "go/build" + "log" + "os" + "path" + "path/filepath" + "regexp" + "strings" +) + +// Context specifies values for operation of ImportPaths that would +// otherwise come from cmd/go/internal/cfg package. +// +// This is a construct added for gotool purposes and doesn't have +// an equivalent upstream in cmd/go. +type Context struct { + // BuildContext is the build context to use. + BuildContext build.Context + + // GOROOTsrc is the location of the src directory in GOROOT. + // At this time, it's used only in MatchPackages to skip + // GOOROOT/src entry from BuildContext.SrcDirs output. + GOROOTsrc string +} + +// allPackages returns all the packages that can be found +// under the $GOPATH directories and $GOROOT matching pattern. +// The pattern is either "all" (all packages), "std" (standard packages), +// "cmd" (standard commands), or a path including "...". +func (c *Context) allPackages(pattern string) []string { + pkgs := c.MatchPackages(pattern) + if len(pkgs) == 0 { + fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) + } + return pkgs +} + +// allPackagesInFS is like allPackages but is passed a pattern +// beginning ./ or ../, meaning it should scan the tree rooted +// at the given directory. There are ... in the pattern too. +func (c *Context) allPackagesInFS(pattern string) []string { + pkgs := c.MatchPackagesInFS(pattern) + if len(pkgs) == 0 { + fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) + } + return pkgs +} + +// MatchPackages returns a list of package paths matching pattern +// (see go help packages for pattern syntax). +func (c *Context) MatchPackages(pattern string) []string { + match := func(string) bool { return true } + treeCanMatch := func(string) bool { return true } + if !IsMetaPackage(pattern) { + match = matchPattern(pattern) + treeCanMatch = treeCanMatchPattern(pattern) + } + + have := map[string]bool{ + "builtin": true, // ignore pseudo-package that exists only for documentation + } + if !c.BuildContext.CgoEnabled { + have["runtime/cgo"] = true // ignore during walk + } + var pkgs []string + + for _, src := range c.BuildContext.SrcDirs() { + if (pattern == "std" || pattern == "cmd") && src != c.GOROOTsrc { + continue + } + src = filepath.Clean(src) + string(filepath.Separator) + root := src + if pattern == "cmd" { + root += "cmd" + string(filepath.Separator) + } + filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + if err != nil || path == src { + return nil + } + + want := true + // Avoid .foo, _foo, and testdata directory trees. + _, elem := filepath.Split(path) + if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" { + want = false + } + + name := filepath.ToSlash(path[len(src):]) + if pattern == "std" && (!isStandardImportPath(name) || name == "cmd") { + // The name "std" is only the standard library. + // If the name is cmd, it's the root of the command tree. + want = false + } + if !treeCanMatch(name) { + want = false + } + + if !fi.IsDir() { + if fi.Mode()&os.ModeSymlink != 0 && want { + if target, err := os.Stat(path); err == nil && target.IsDir() { + fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) + } + } + return nil + } + if !want { + return filepath.SkipDir + } + + if have[name] { + return nil + } + have[name] = true + if !match(name) { + return nil + } + pkg, err := c.BuildContext.ImportDir(path, 0) + if err != nil { + if _, noGo := err.(*build.NoGoError); noGo { + return nil + } + } + + // If we are expanding "cmd", skip main + // packages under cmd/vendor. At least as of + // March, 2017, there is one there for the + // vendored pprof tool. + if pattern == "cmd" && strings.HasPrefix(pkg.ImportPath, "cmd/vendor") && pkg.Name == "main" { + return nil + } + + pkgs = append(pkgs, name) + return nil + }) + } + return pkgs +} + +// MatchPackagesInFS returns a list of package paths matching pattern, +// which must begin with ./ or ../ +// (see go help packages for pattern syntax). +func (c *Context) MatchPackagesInFS(pattern string) []string { + // Find directory to begin the scan. + // Could be smarter but this one optimization + // is enough for now, since ... is usually at the + // end of a path. + i := strings.Index(pattern, "...") + dir, _ := path.Split(pattern[:i]) + + // pattern begins with ./ or ../. + // path.Clean will discard the ./ but not the ../. + // We need to preserve the ./ for pattern matching + // and in the returned import paths. + prefix := "" + if strings.HasPrefix(pattern, "./") { + prefix = "./" + } + match := matchPattern(pattern) + + var pkgs []string + filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { + if err != nil || !fi.IsDir() { + return nil + } + if path == dir { + // filepath.Walk starts at dir and recurses. For the recursive case, + // the path is the result of filepath.Join, which calls filepath.Clean. + // The initial case is not Cleaned, though, so we do this explicitly. + // + // This converts a path like "./io/" to "io". Without this step, running + // "cd $GOROOT/src; go list ./io/..." would incorrectly skip the io + // package, because prepending the prefix "./" to the unclean path would + // result in "././io", and match("././io") returns false. + path = filepath.Clean(path) + } + + // Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..". + _, elem := filepath.Split(path) + dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".." + if dot || strings.HasPrefix(elem, "_") || elem == "testdata" { + return filepath.SkipDir + } + + name := prefix + filepath.ToSlash(path) + if !match(name) { + return nil + } + + // We keep the directory if we can import it, or if we can't import it + // due to invalid Go source files. This means that directories containing + // parse errors will be built (and fail) instead of being silently skipped + // as not matching the pattern. Go 1.5 and earlier skipped, but that + // behavior means people miss serious mistakes. + // See golang.org/issue/11407. + if p, err := c.BuildContext.ImportDir(path, 0); err != nil && (p == nil || len(p.InvalidGoFiles) == 0) { + if _, noGo := err.(*build.NoGoError); !noGo { + log.Print(err) + } + return nil + } + pkgs = append(pkgs, name) + return nil + }) + return pkgs +} + +// treeCanMatchPattern(pattern)(name) reports whether +// name or children of name can possibly match pattern. +// Pattern is the same limited glob accepted by matchPattern. +func treeCanMatchPattern(pattern string) func(name string) bool { + wildCard := false + if i := strings.Index(pattern, "..."); i >= 0 { + wildCard = true + pattern = pattern[:i] + } + return func(name string) bool { + return len(name) <= len(pattern) && hasPathPrefix(pattern, name) || + wildCard && strings.HasPrefix(name, pattern) + } +} + +// matchPattern(pattern)(name) reports whether +// name matches pattern. Pattern is a limited glob +// pattern in which '...' means 'any string' and there +// is no other special syntax. +// Unfortunately, there are two special cases. Quoting "go help packages": +// +// First, /... at the end of the pattern can match an empty string, +// so that net/... matches both net and packages in its subdirectories, like net/http. +// Second, any slash-separted pattern element containing a wildcard never +// participates in a match of the "vendor" element in the path of a vendored +// package, so that ./... does not match packages in subdirectories of +// ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do. +// Note, however, that a directory named vendor that itself contains code +// is not a vendored package: cmd/vendor would be a command named vendor, +// and the pattern cmd/... matches it. +func matchPattern(pattern string) func(name string) bool { + // Convert pattern to regular expression. + // The strategy for the trailing /... is to nest it in an explicit ? expression. + // The strategy for the vendor exclusion is to change the unmatchable + // vendor strings to a disallowed code point (vendorChar) and to use + // "(anything but that codepoint)*" as the implementation of the ... wildcard. + // This is a bit complicated but the obvious alternative, + // namely a hand-written search like in most shell glob matchers, + // is too easy to make accidentally exponential. + // Using package regexp guarantees linear-time matching. + + const vendorChar = "\x00" + + if strings.Contains(pattern, vendorChar) { + return func(name string) bool { return false } + } + + re := regexp.QuoteMeta(pattern) + re = replaceVendor(re, vendorChar) + switch { + case strings.HasSuffix(re, `/`+vendorChar+`/\.\.\.`): + re = strings.TrimSuffix(re, `/`+vendorChar+`/\.\.\.`) + `(/vendor|/` + vendorChar + `/\.\.\.)` + case re == vendorChar+`/\.\.\.`: + re = `(/vendor|/` + vendorChar + `/\.\.\.)` + case strings.HasSuffix(re, `/\.\.\.`): + re = strings.TrimSuffix(re, `/\.\.\.`) + `(/\.\.\.)?` + } + re = strings.Replace(re, `\.\.\.`, `[^`+vendorChar+`]*`, -1) + + reg := regexp.MustCompile(`^` + re + `$`) + + return func(name string) bool { + if strings.Contains(name, vendorChar) { + return false + } + return reg.MatchString(replaceVendor(name, vendorChar)) + } +} + +// replaceVendor returns the result of replacing +// non-trailing vendor path elements in x with repl. +func replaceVendor(x, repl string) string { + if !strings.Contains(x, "vendor") { + return x + } + elem := strings.Split(x, "/") + for i := 0; i < len(elem)-1; i++ { + if elem[i] == "vendor" { + elem[i] = repl + } + } + return strings.Join(elem, "/") +} + +// ImportPaths returns the import paths to use for the given command line. +func (c *Context) ImportPaths(args []string) []string { + args = c.ImportPathsNoDotExpansion(args) + var out []string + for _, a := range args { + if strings.Contains(a, "...") { + if build.IsLocalImport(a) { + out = append(out, c.allPackagesInFS(a)...) + } else { + out = append(out, c.allPackages(a)...) + } + continue + } + out = append(out, a) + } + return out +} + +// ImportPathsNoDotExpansion returns the import paths to use for the given +// command line, but it does no ... expansion. +func (c *Context) ImportPathsNoDotExpansion(args []string) []string { + if len(args) == 0 { + return []string{"."} + } + var out []string + for _, a := range args { + // Arguments are supposed to be import paths, but + // as a courtesy to Windows developers, rewrite \ to / + // in command-line arguments. Handles .\... and so on. + if filepath.Separator == '\\' { + a = strings.Replace(a, `\`, `/`, -1) + } + + // Put argument in canonical form, but preserve leading ./. + if strings.HasPrefix(a, "./") { + a = "./" + path.Clean(a) + if a == "./." { + a = "." + } + } else { + a = path.Clean(a) + } + if IsMetaPackage(a) { + out = append(out, c.allPackages(a)...) + continue + } + out = append(out, a) + } + return out +} + +// IsMetaPackage checks if name is a reserved package name that expands to multiple packages. +func IsMetaPackage(name string) bool { + return name == "std" || name == "cmd" || name == "all" +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/match.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/match.go index e9209b7e8..4dbdbff47 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/match.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/match.go @@ -26,285 +26,31 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +build go1.9 + package gotool import ( - "fmt" - "go/build" - "log" - "os" - "path" "path/filepath" - "regexp" - "strings" + + "github.com/kisielk/gotool/internal/load" ) -// This file contains code from the Go distribution. - -// matchPattern(pattern)(name) reports whether -// name matches pattern. Pattern is a limited glob -// pattern in which '...' means 'any string' and there -// is no other special syntax. -func matchPattern(pattern string) func(name string) bool { - re := regexp.QuoteMeta(pattern) - re = strings.Replace(re, `\.\.\.`, `.*`, -1) - // Special case: foo/... matches foo too. - if strings.HasSuffix(re, `/.*`) { - re = re[:len(re)-len(`/.*`)] + `(/.*)?` - } - reg := regexp.MustCompile(`^` + re + `$`) - return reg.MatchString -} - -func (c *Context) matchPackages(pattern string) []string { - match := func(string) bool { return true } - treeCanMatch := func(string) bool { return true } - if !isMetaPackage(pattern) { - match = matchPattern(pattern) - treeCanMatch = treeCanMatchPattern(pattern) - } - - have := map[string]bool{ - "builtin": true, // ignore pseudo-package that exists only for documentation - } - if !c.BuildContext.CgoEnabled { - have["runtime/cgo"] = true // ignore during walk - } - var pkgs []string - - for _, src := range c.BuildContext.SrcDirs() { - if (pattern == "std" || pattern == "cmd") && src != gorootSrc { - continue - } - src = filepath.Clean(src) + string(filepath.Separator) - root := src - if pattern == "cmd" { - root += "cmd" + string(filepath.Separator) - } - filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { - if err != nil || !fi.IsDir() || path == src { - return nil - } - - // Avoid .foo, _foo, and testdata directory trees. - _, elem := filepath.Split(path) - if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" { - return filepath.SkipDir - } - - name := filepath.ToSlash(path[len(src):]) - if pattern == "std" && (!isStandardImportPath(name) || name == "cmd") { - // The name "std" is only the standard library. - // If the name is cmd, it's the root of the command tree. - return filepath.SkipDir - } - if !treeCanMatch(name) { - return filepath.SkipDir - } - if have[name] { - return nil - } - have[name] = true - if !match(name) { - return nil - } - _, err = c.BuildContext.ImportDir(path, 0) - if err != nil { - if _, noGo := err.(*build.NoGoError); noGo { - return nil - } - } - pkgs = append(pkgs, name) - return nil - }) - } - return pkgs -} - -// importPathsNoDotExpansion returns the import paths to use for the given -// command line, but it does no ... expansion. -func (c *Context) importPathsNoDotExpansion(args []string) []string { - if len(args) == 0 { - return []string{"."} - } - var out []string - for _, a := range args { - // Arguments are supposed to be import paths, but - // as a courtesy to Windows developers, rewrite \ to / - // in command-line arguments. Handles .\... and so on. - if filepath.Separator == '\\' { - a = strings.Replace(a, `\`, `/`, -1) - } - - // Put argument in canonical form, but preserve leading ./. - if strings.HasPrefix(a, "./") { - a = "./" + path.Clean(a) - if a == "./." { - a = "." - } - } else { - a = path.Clean(a) - } - if isMetaPackage(a) { - out = append(out, c.allPackages(a)...) - continue - } - out = append(out, a) - } - return out -} - // importPaths returns the import paths to use for the given command line. func (c *Context) importPaths(args []string) []string { - args = c.importPathsNoDotExpansion(args) - var out []string - for _, a := range args { - if strings.Contains(a, "...") { - if build.IsLocalImport(a) { - out = append(out, c.allPackagesInFS(a)...) - } else { - out = append(out, c.allPackages(a)...) - } - continue - } - out = append(out, a) + lctx := load.Context{ + BuildContext: c.BuildContext, + GOROOTsrc: c.joinPath(c.BuildContext.GOROOT, "src"), } - return out + return lctx.ImportPaths(args) } -// allPackages returns all the packages that can be found -// under the $GOPATH directories and $GOROOT matching pattern. -// The pattern is either "all" (all packages), "std" (standard packages), -// "cmd" (standard commands), or a path including "...". -func (c *Context) allPackages(pattern string) []string { - pkgs := c.matchPackages(pattern) - if len(pkgs) == 0 { - fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) - } - return pkgs -} - -// allPackagesInFS is like allPackages but is passed a pattern -// beginning ./ or ../, meaning it should scan the tree rooted -// at the given directory. There are ... in the pattern too. -func (c *Context) allPackagesInFS(pattern string) []string { - pkgs := c.matchPackagesInFS(pattern) - if len(pkgs) == 0 { - fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) - } - return pkgs -} - -func (c *Context) matchPackagesInFS(pattern string) []string { - // Find directory to begin the scan. - // Could be smarter but this one optimization - // is enough for now, since ... is usually at the - // end of a path. - i := strings.Index(pattern, "...") - dir, _ := path.Split(pattern[:i]) - - // pattern begins with ./ or ../. - // path.Clean will discard the ./ but not the ../. - // We need to preserve the ./ for pattern matching - // and in the returned import paths. - prefix := "" - if strings.HasPrefix(pattern, "./") { - prefix = "./" - } - match := matchPattern(pattern) - - var pkgs []string - filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { - if err != nil || !fi.IsDir() { - return nil - } - if path == dir { - // filepath.Walk starts at dir and recurses. For the recursive case, - // the path is the result of filepath.Join, which calls filepath.Clean. - // The initial case is not Cleaned, though, so we do this explicitly. - // - // This converts a path like "./io/" to "io". Without this step, running - // "cd $GOROOT/src; go list ./io/..." would incorrectly skip the io - // package, because prepending the prefix "./" to the unclean path would - // result in "././io", and match("././io") returns false. - path = filepath.Clean(path) - } - - // Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..". - _, elem := filepath.Split(path) - dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".." - if dot || strings.HasPrefix(elem, "_") || elem == "testdata" { - return filepath.SkipDir - } - - name := prefix + filepath.ToSlash(path) - if !match(name) { - return nil - } - - // We keep the directory if we can import it, or if we can't import it - // due to invalid Go source files. This means that directories containing - // parse errors will be built (and fail) instead of being silently skipped - // as not matching the pattern. Go 1.5 and earlier skipped, but that - // behavior means people miss serious mistakes. - // See golang.org/issue/11407. - if p, err := c.BuildContext.ImportDir(path, 0); err != nil && shouldIgnoreImport(p) { - if _, noGo := err.(*build.NoGoError); !noGo { - log.Print(err) - } - return nil - } - pkgs = append(pkgs, name) - return nil - }) - return pkgs -} - -// isMetaPackage checks if name is a reserved package name that expands to multiple packages -func isMetaPackage(name string) bool { - return name == "std" || name == "cmd" || name == "all" -} - -// isStandardImportPath reports whether $GOROOT/src/path should be considered -// part of the standard distribution. For historical reasons we allow people to add -// their own code to $GOROOT instead of using $GOPATH, but we assume that -// code will start with a domain name (dot in the first element). -func isStandardImportPath(path string) bool { - i := strings.Index(path, "/") - if i < 0 { - i = len(path) - } - elem := path[:i] - return !strings.Contains(elem, ".") -} - -// hasPathPrefix reports whether the path s begins with the -// elements in prefix. -func hasPathPrefix(s, prefix string) bool { - switch { - default: - return false - case len(s) == len(prefix): - return s == prefix - case len(s) > len(prefix): - if prefix != "" && prefix[len(prefix)-1] == '/' { - return strings.HasPrefix(s, prefix) - } - return s[len(prefix)] == '/' && s[:len(prefix)] == prefix - } -} - -// treeCanMatchPattern(pattern)(name) reports whether -// name or children of name can possibly match pattern. -// Pattern is the same limited glob accepted by matchPattern. -func treeCanMatchPattern(pattern string) func(name string) bool { - wildCard := false - if i := strings.Index(pattern, "..."); i >= 0 { - wildCard = true - pattern = pattern[:i] - } - return func(name string) bool { - return len(name) <= len(pattern) && hasPathPrefix(pattern, name) || - wildCard && strings.HasPrefix(name, pattern) +// joinPath calls c.BuildContext.JoinPath (if not nil) or else filepath.Join. +// +// It's a copy of the unexported build.Context.joinPath helper. +func (c *Context) joinPath(elem ...string) string { + if f := c.BuildContext.JoinPath; f != nil { + return f(elem...) } + return filepath.Join(elem...) } diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/match18.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/match18.go new file mode 100644 index 000000000..6d6b1368c --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/kisielk/gotool/match18.go @@ -0,0 +1,317 @@ +// Copyright (c) 2009 The Go Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// +build !go1.9 + +package gotool + +import ( + "fmt" + "go/build" + "log" + "os" + "path" + "path/filepath" + "regexp" + "strings" +) + +// This file contains code from the Go distribution. + +// matchPattern(pattern)(name) reports whether +// name matches pattern. Pattern is a limited glob +// pattern in which '...' means 'any string' and there +// is no other special syntax. +func matchPattern(pattern string) func(name string) bool { + re := regexp.QuoteMeta(pattern) + re = strings.Replace(re, `\.\.\.`, `.*`, -1) + // Special case: foo/... matches foo too. + if strings.HasSuffix(re, `/.*`) { + re = re[:len(re)-len(`/.*`)] + `(/.*)?` + } + reg := regexp.MustCompile(`^` + re + `$`) + return reg.MatchString +} + +// matchPackages returns a list of package paths matching pattern +// (see go help packages for pattern syntax). +func (c *Context) matchPackages(pattern string) []string { + match := func(string) bool { return true } + treeCanMatch := func(string) bool { return true } + if !isMetaPackage(pattern) { + match = matchPattern(pattern) + treeCanMatch = treeCanMatchPattern(pattern) + } + + have := map[string]bool{ + "builtin": true, // ignore pseudo-package that exists only for documentation + } + if !c.BuildContext.CgoEnabled { + have["runtime/cgo"] = true // ignore during walk + } + var pkgs []string + + for _, src := range c.BuildContext.SrcDirs() { + if (pattern == "std" || pattern == "cmd") && src != gorootSrc { + continue + } + src = filepath.Clean(src) + string(filepath.Separator) + root := src + if pattern == "cmd" { + root += "cmd" + string(filepath.Separator) + } + filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + if err != nil || !fi.IsDir() || path == src { + return nil + } + + // Avoid .foo, _foo, and testdata directory trees. + _, elem := filepath.Split(path) + if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" { + return filepath.SkipDir + } + + name := filepath.ToSlash(path[len(src):]) + if pattern == "std" && (!isStandardImportPath(name) || name == "cmd") { + // The name "std" is only the standard library. + // If the name is cmd, it's the root of the command tree. + return filepath.SkipDir + } + if !treeCanMatch(name) { + return filepath.SkipDir + } + if have[name] { + return nil + } + have[name] = true + if !match(name) { + return nil + } + _, err = c.BuildContext.ImportDir(path, 0) + if err != nil { + if _, noGo := err.(*build.NoGoError); noGo { + return nil + } + } + pkgs = append(pkgs, name) + return nil + }) + } + return pkgs +} + +// importPathsNoDotExpansion returns the import paths to use for the given +// command line, but it does no ... expansion. +func (c *Context) importPathsNoDotExpansion(args []string) []string { + if len(args) == 0 { + return []string{"."} + } + var out []string + for _, a := range args { + // Arguments are supposed to be import paths, but + // as a courtesy to Windows developers, rewrite \ to / + // in command-line arguments. Handles .\... and so on. + if filepath.Separator == '\\' { + a = strings.Replace(a, `\`, `/`, -1) + } + + // Put argument in canonical form, but preserve leading ./. + if strings.HasPrefix(a, "./") { + a = "./" + path.Clean(a) + if a == "./." { + a = "." + } + } else { + a = path.Clean(a) + } + if isMetaPackage(a) { + out = append(out, c.allPackages(a)...) + continue + } + out = append(out, a) + } + return out +} + +// importPaths returns the import paths to use for the given command line. +func (c *Context) importPaths(args []string) []string { + args = c.importPathsNoDotExpansion(args) + var out []string + for _, a := range args { + if strings.Contains(a, "...") { + if build.IsLocalImport(a) { + out = append(out, c.allPackagesInFS(a)...) + } else { + out = append(out, c.allPackages(a)...) + } + continue + } + out = append(out, a) + } + return out +} + +// allPackages returns all the packages that can be found +// under the $GOPATH directories and $GOROOT matching pattern. +// The pattern is either "all" (all packages), "std" (standard packages), +// "cmd" (standard commands), or a path including "...". +func (c *Context) allPackages(pattern string) []string { + pkgs := c.matchPackages(pattern) + if len(pkgs) == 0 { + fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) + } + return pkgs +} + +// allPackagesInFS is like allPackages but is passed a pattern +// beginning ./ or ../, meaning it should scan the tree rooted +// at the given directory. There are ... in the pattern too. +func (c *Context) allPackagesInFS(pattern string) []string { + pkgs := c.matchPackagesInFS(pattern) + if len(pkgs) == 0 { + fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) + } + return pkgs +} + +// matchPackagesInFS returns a list of package paths matching pattern, +// which must begin with ./ or ../ +// (see go help packages for pattern syntax). +func (c *Context) matchPackagesInFS(pattern string) []string { + // Find directory to begin the scan. + // Could be smarter but this one optimization + // is enough for now, since ... is usually at the + // end of a path. + i := strings.Index(pattern, "...") + dir, _ := path.Split(pattern[:i]) + + // pattern begins with ./ or ../. + // path.Clean will discard the ./ but not the ../. + // We need to preserve the ./ for pattern matching + // and in the returned import paths. + prefix := "" + if strings.HasPrefix(pattern, "./") { + prefix = "./" + } + match := matchPattern(pattern) + + var pkgs []string + filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { + if err != nil || !fi.IsDir() { + return nil + } + if path == dir { + // filepath.Walk starts at dir and recurses. For the recursive case, + // the path is the result of filepath.Join, which calls filepath.Clean. + // The initial case is not Cleaned, though, so we do this explicitly. + // + // This converts a path like "./io/" to "io". Without this step, running + // "cd $GOROOT/src; go list ./io/..." would incorrectly skip the io + // package, because prepending the prefix "./" to the unclean path would + // result in "././io", and match("././io") returns false. + path = filepath.Clean(path) + } + + // Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..". + _, elem := filepath.Split(path) + dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".." + if dot || strings.HasPrefix(elem, "_") || elem == "testdata" { + return filepath.SkipDir + } + + name := prefix + filepath.ToSlash(path) + if !match(name) { + return nil + } + + // We keep the directory if we can import it, or if we can't import it + // due to invalid Go source files. This means that directories containing + // parse errors will be built (and fail) instead of being silently skipped + // as not matching the pattern. Go 1.5 and earlier skipped, but that + // behavior means people miss serious mistakes. + // See golang.org/issue/11407. + if p, err := c.BuildContext.ImportDir(path, 0); err != nil && shouldIgnoreImport(p) { + if _, noGo := err.(*build.NoGoError); !noGo { + log.Print(err) + } + return nil + } + pkgs = append(pkgs, name) + return nil + }) + return pkgs +} + +// isMetaPackage checks if name is a reserved package name that expands to multiple packages. +func isMetaPackage(name string) bool { + return name == "std" || name == "cmd" || name == "all" +} + +// isStandardImportPath reports whether $GOROOT/src/path should be considered +// part of the standard distribution. For historical reasons we allow people to add +// their own code to $GOROOT instead of using $GOPATH, but we assume that +// code will start with a domain name (dot in the first element). +func isStandardImportPath(path string) bool { + i := strings.Index(path, "/") + if i < 0 { + i = len(path) + } + elem := path[:i] + return !strings.Contains(elem, ".") +} + +// hasPathPrefix reports whether the path s begins with the +// elements in prefix. +func hasPathPrefix(s, prefix string) bool { + switch { + default: + return false + case len(s) == len(prefix): + return s == prefix + case len(s) > len(prefix): + if prefix != "" && prefix[len(prefix)-1] == '/' { + return strings.HasPrefix(s, prefix) + } + return s[len(prefix)] == '/' && s[:len(prefix)] == prefix + } +} + +// treeCanMatchPattern(pattern)(name) reports whether +// name or children of name can possibly match pattern. +// Pattern is the same limited glob accepted by matchPattern. +func treeCanMatchPattern(pattern string) func(name string) bool { + wildCard := false + if i := strings.Index(pattern, "..."); i >= 0 { + wildCard = true + pattern = pattern[:i] + } + return func(name string) bool { + return len(name) <= len(pattern) && hasPathPrefix(pattern, name) || + wildCard && strings.HasPrefix(name, pattern) + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mdempsky/maligned/LICENSE b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mdempsky/maligned/LICENSE new file mode 100644 index 000000000..744875676 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mdempsky/maligned/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mdempsky/maligned/maligned.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mdempsky/maligned/maligned.go new file mode 100644 index 000000000..ae3fadf67 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mdempsky/maligned/maligned.go @@ -0,0 +1,229 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + "go/ast" + "go/build" + "go/token" + "go/types" + "log" + "sort" + + "github.com/kisielk/gotool" + "golang.org/x/tools/go/loader" +) + +var fset = token.NewFileSet() + +func main() { + flag.Parse() + + importPaths := gotool.ImportPaths(flag.Args()) + if len(importPaths) == 0 { + return + } + + var conf loader.Config + conf.Fset = fset + for _, importPath := range importPaths { + conf.Import(importPath) + } + prog, err := conf.Load() + if err != nil { + log.Fatal(err) + } + + for _, pkg := range prog.InitialPackages() { + for _, file := range pkg.Files { + ast.Inspect(file, func(node ast.Node) bool { + if s, ok := node.(*ast.StructType); ok { + malign(node.Pos(), pkg.Types[s].Type.(*types.Struct)) + } + return true + }) + } + } +} + +func malign(pos token.Pos, str *types.Struct) { + wordSize := int64(8) + maxAlign := int64(8) + switch build.Default.GOARCH { + case "386", "arm": + wordSize, maxAlign = 4, 4 + case "amd64p32": + wordSize = 4 + } + + s := gcSizes{wordSize, maxAlign} + sz, opt := s.Sizeof(str), optimalSize(str, &s) + if sz != opt { + fmt.Printf("%s: struct of size %d could be %d\n", fset.Position(pos), sz, opt) + } +} + +func optimalSize(str *types.Struct, sizes *gcSizes) int64 { + nf := str.NumFields() + fields := make([]*types.Var, nf) + alignofs := make([]int64, nf) + sizeofs := make([]int64, nf) + for i := 0; i < nf; i++ { + fields[i] = str.Field(i) + ft := fields[i].Type() + alignofs[i] = sizes.Alignof(ft) + sizeofs[i] = sizes.Sizeof(ft) + } + sort.Sort(&byAlignAndSize{fields, alignofs, sizeofs}) + return sizes.Sizeof(types.NewStruct(fields, nil)) +} + +type byAlignAndSize struct { + fields []*types.Var + alignofs []int64 + sizeofs []int64 +} + +func (s *byAlignAndSize) Len() int { return len(s.fields) } +func (s *byAlignAndSize) Swap(i, j int) { + s.fields[i], s.fields[j] = s.fields[j], s.fields[i] + s.alignofs[i], s.alignofs[j] = s.alignofs[j], s.alignofs[i] + s.sizeofs[i], s.sizeofs[j] = s.sizeofs[j], s.sizeofs[i] +} + +func (s *byAlignAndSize) Less(i, j int) bool { + // Place zero sized objects before non-zero sized objects. + if s.sizeofs[i] == 0 && s.sizeofs[j] != 0 { + return true + } + if s.sizeofs[j] == 0 && s.sizeofs[i] != 0 { + return false + } + + // Next, place more tightly aligned objects before less tightly aligned objects. + if s.alignofs[i] != s.alignofs[j] { + return s.alignofs[i] > s.alignofs[j] + } + + // Lastly, order by size. + if s.sizeofs[i] != s.sizeofs[j] { + return s.sizeofs[i] > s.sizeofs[j] + } + + return false +} + +// Code below based on go/types.StdSizes. + +type gcSizes struct { + WordSize int64 + MaxAlign int64 +} + +func (s *gcSizes) Alignof(T types.Type) int64 { + // NOTE: On amd64, complex64 is 8 byte aligned, + // even though float32 is only 4 byte aligned. + + // For arrays and structs, alignment is defined in terms + // of alignment of the elements and fields, respectively. + switch t := T.Underlying().(type) { + case *types.Array: + // spec: "For a variable x of array type: unsafe.Alignof(x) + // is the same as unsafe.Alignof(x[0]), but at least 1." + return s.Alignof(t.Elem()) + case *types.Struct: + // spec: "For a variable x of struct type: unsafe.Alignof(x) + // is the largest of the values unsafe.Alignof(x.f) for each + // field f of x, but at least 1." + max := int64(1) + for i, nf := 0, t.NumFields(); i < nf; i++ { + if a := s.Alignof(t.Field(i).Type()); a > max { + max = a + } + } + return max + } + a := s.Sizeof(T) // may be 0 + // spec: "For a variable x of any type: unsafe.Alignof(x) is at least 1." + if a < 1 { + return 1 + } + if a > s.MaxAlign { + return s.MaxAlign + } + return a +} + +var basicSizes = [...]byte{ + types.Bool: 1, + types.Int8: 1, + types.Int16: 2, + types.Int32: 4, + types.Int64: 8, + types.Uint8: 1, + types.Uint16: 2, + types.Uint32: 4, + types.Uint64: 8, + types.Float32: 4, + types.Float64: 8, + types.Complex64: 8, + types.Complex128: 16, +} + +func (s *gcSizes) Sizeof(T types.Type) int64 { + switch t := T.Underlying().(type) { + case *types.Basic: + k := t.Kind() + if int(k) < len(basicSizes) { + if s := basicSizes[k]; s > 0 { + return int64(s) + } + } + if k == types.String { + return s.WordSize * 2 + } + case *types.Array: + n := t.Len() + if n == 0 { + return 0 + } + a := s.Alignof(t.Elem()) + z := s.Sizeof(t.Elem()) + return align(z, a)*(n-1) + z + case *types.Slice: + return s.WordSize * 3 + case *types.Struct: + nf := t.NumFields() + if nf == 0 { + return 0 + } + + var o int64 + max := int64(1) + for i := 0; i < nf; i++ { + ft := t.Field(i).Type() + a, sz := s.Alignof(ft), s.Sizeof(ft) + if a > max { + max = a + } + if i == nf-1 && sz == 0 && o != 0 { + sz = 1 + } + o = align(o, a) + sz + } + return align(o, max) + case *types.Interface: + return s.WordSize * 2 + } + return s.WordSize // catch-all +} + +// align returns the smallest y >= x such that y % a == 0. +func align(x, a int64) int64 { + y := x + a - 1 + return y - y%a +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/unparam/main.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/unparam/main.go deleted file mode 100644 index 801748bbd..000000000 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/unparam/main.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2017, Daniel Martí -// See LICENSE for licensing information - -package main - -import ( - "flag" - "fmt" - "os" - - "github.com/mvdan/unparam/check" -) - -var tests = flag.Bool("tests", true, "include tests") - -func main() { - flag.Parse() - warns, err := check.UnusedParams(*tests, flag.Args()...) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - for _, warn := range warns { - fmt.Println(warn) - } -} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/container/intsets/sparse.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/container/intsets/sparse.go index 8847febf1..5db01c1a4 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/container/intsets/sparse.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/container/intsets/sparse.go @@ -21,10 +21,6 @@ package intsets // import "golang.org/x/tools/container/intsets" // The space usage would be proportional to Max(), not Len(), and the // implementation would be based upon big.Int. // -// TODO(adonovan): experiment with making the root block indirect (nil -// iff IsEmpty). This would reduce the memory usage when empty and -// might simplify the aliasing invariants. -// // TODO(adonovan): opt: make UnionWith and Difference faster. // These are the hot-spots for go/pointer. @@ -45,9 +41,10 @@ type Sparse struct { // An uninitialized Sparse represents an empty set. // An empty set may also be represented by // root.next == root.prev == &root. - // In a non-empty set, root.next points to the first block and - // root.prev to the last. - // root.offset and root.bits are unused. + // + // The root is always the block with the smallest offset. + // It can be empty, but only if it is the only block; in that case, offset is + // MaxInt (which is not a valid offset). root block } @@ -144,7 +141,6 @@ func (b *block) len() int { // max returns the maximum element of the block. // The block must not be empty. -// func (b *block) max() int { bi := b.offset + bitsPerBlock // Decrement bi by number of high zeros in last.bits. @@ -161,7 +157,6 @@ func (b *block) max() int { // and also removes it if take is set. // The block must not be initially empty. // NB: may leave the block empty. -// func (b *block) min(take bool) int { for i, w := range b.bits { if w != 0 { @@ -175,6 +170,26 @@ func (b *block) min(take bool) int { panic("BUG: empty block") } +// lowerBound returns the smallest element of the block that is greater than or +// equal to the element corresponding to the ith bit. If there is no such +// element, the second return value is false. +func (b *block) lowerBound(i uint) (int, bool) { + w := i / bitsPerWord + bit := i % bitsPerWord + + if val := b.bits[w] >> bit; val != 0 { + return b.offset + int(i) + ntz(val), true + } + + for w++; w < wordsPerBlock; w++ { + if val := b.bits[w]; val != 0 { + return b.offset + int(w*bitsPerWord) + ntz(val), true + } + } + + return 0, false +} + // forEach calls f for each element of block b. // f must not mutate b's enclosing Sparse. func (b *block) forEach(f func(int)) { @@ -204,14 +219,20 @@ func offsetAndBitIndex(x int) (int, uint) { // -- Sparse -------------------------------------------------------------- -// start returns the root's next block, which is the root block -// (if s.IsEmpty()) or the first true block otherwise. -// start has the side effect of ensuring that s is properly -// initialized. -// -func (s *Sparse) start() *block { +// none is a shared, empty, sentinel block that indicates the end of a block +// list. +var none block + +// Dummy type used to generate an implicit panic. This must be defined at the +// package level; if it is defined inside a function, it prevents the inlining +// of that function. +type to_copy_a_sparse_you_must_call_its_Copy_method struct{} + +// init ensures s is properly initialized. +func (s *Sparse) init() { root := &s.root if root.next == nil { + root.offset = MaxInt root.next = root root.prev = root } else if root.next.prev != root { @@ -219,21 +240,45 @@ func (s *Sparse) start() *block { // new Sparse y shares the old linked list, but iteration // on y will never encounter &y.root so it goes into a // loop. Fail fast before this occurs. - panic("A Sparse has been copied without (*Sparse).Copy()") + // We don't want to call panic here because it prevents the + // inlining of this function. + _ = (interface{}(nil)).(to_copy_a_sparse_you_must_call_its_Copy_method) } +} - return root.next +func (s *Sparse) first() *block { + s.init() + if s.root.offset == MaxInt { + return &none + } + return &s.root +} + +// next returns the next block in the list, or end if b is the last block. +func (s *Sparse) next(b *block) *block { + if b.next == &s.root { + return &none + } + return b.next +} + +// prev returns the previous block in the list, or end if b is the first block. +func (s *Sparse) prev(b *block) *block { + if b.prev == &s.root { + return &none + } + return b.prev } // IsEmpty reports whether the set s is empty. func (s *Sparse) IsEmpty() bool { - return s.start() == &s.root + return s.root.next == nil || s.root.offset == MaxInt } // Len returns the number of elements in the set s. func (s *Sparse) Len() int { var l int - for b := s.start(); b != &s.root; b = b.next { + for b := s.first(); b != &none; b = s.next(b) { l += b.len() } return l @@ -252,19 +297,34 @@ func (s *Sparse) Min() int { if s.IsEmpty() { return MaxInt } - return s.root.next.min(false) + return s.root.min(false) +} + +// LowerBound returns the smallest element >= x, or MaxInt if there is no such +// element. +func (s *Sparse) LowerBound(x int) int { + offset, i := offsetAndBitIndex(x) + for b := s.first(); b != &none; b = s.next(b) { + if b.offset > offset { + return b.min(false) + } + if b.offset == offset { + if y, ok := b.lowerBound(i); ok { + return y + } + } + } + return MaxInt } // block returns the block that would contain offset, // or nil if s contains no such block. -// +// Precondition: offset is a multiple of bitsPerBlock. func (s *Sparse) block(offset int) *block { - b := s.start() - for b != &s.root && b.offset <= offset { + for b := s.first(); b != &none && b.offset <= offset; b = s.next(b) { if b.offset == offset { return b } - b = b.next } return nil } @@ -272,26 +332,49 @@ func (s *Sparse) block(offset int) *block { // Insert adds x to the set s, and reports whether the set grew. func (s *Sparse) Insert(x int) bool { offset, i := offsetAndBitIndex(x) - b := s.start() - for b != &s.root && b.offset <= offset { + + b := s.first() + for ; b != &none && b.offset <= offset; b = s.next(b) { if b.offset == offset { return b.insert(i) } - b = b.next } // Insert new block before b. - new := &block{offset: offset} - new.next = b - new.prev = b.prev - new.prev.next = new - new.next.prev = new + new := s.insertBlockBefore(b) + new.offset = offset return new.insert(i) } -func (s *Sparse) removeBlock(b *block) { - b.prev.next = b.next - b.next.prev = b.prev +// removeBlock removes a block and returns the block that followed it (or end if +// it was the last block). +func (s *Sparse) removeBlock(b *block) *block { + if b != &s.root { + b.prev.next = b.next + b.next.prev = b.prev + if b.next == &s.root { + return &none + } + return b.next + } + + first := s.root.next + if first == &s.root { + // This was the only block. + s.Clear() + return &none + } + s.root.offset = first.offset + s.root.bits = first.bits + if first.next == &s.root { + // Single block remaining. + s.root.next = &s.root + s.root.prev = &s.root + } else { + s.root.next = first.next + first.next.prev = &s.root + } + return &s.root } // Remove removes x from the set s, and reports whether the set shrank. @@ -311,8 +394,11 @@ func (s *Sparse) Remove(x int) bool { // Clear removes all elements from the set s. func (s *Sparse) Clear() { - s.root.next = &s.root - s.root.prev = &s.root + s.root = block{ + offset: MaxInt, + next: &s.root, + prev: &s.root, + } } // If set s is non-empty, TakeMin sets *p to the minimum element of @@ -325,13 +411,12 @@ func (s *Sparse) Clear() { // for worklist.TakeMin(&x) { use(x) } // func (s *Sparse) TakeMin(p *int) bool { - head := s.start() - if head == &s.root { + if s.IsEmpty() { return false } - *p = head.min(true) - if head.empty() { - s.removeBlock(head) + *p = s.root.min(true) + if s.root.empty() { + s.removeBlock(&s.root) } return true } @@ -352,7 +437,7 @@ func (s *Sparse) Has(x int) bool { // natural control flow with continue/break/return. // func (s *Sparse) forEach(f func(int)) { - for b := s.start(); b != &s.root; b = b.next { + for b := s.first(); b != &none; b = s.next(b) { b.forEach(f) } } @@ -363,22 +448,51 @@ func (s *Sparse) Copy(x *Sparse) { return } - xb := x.start() - sb := s.start() - for xb != &x.root { - if sb == &s.root { + xb := x.first() + sb := s.first() + for xb != &none { + if sb == &none { sb = s.insertBlockBefore(sb) } sb.offset = xb.offset sb.bits = xb.bits - xb = xb.next - sb = sb.next + xb = x.next(xb) + sb = s.next(sb) } s.discardTail(sb) } // insertBlockBefore returns a new block, inserting it before next. +// If next is the root, the root is replaced. If next is end, the block is +// inserted at the end. func (s *Sparse) insertBlockBefore(next *block) *block { + if s.IsEmpty() { + if next != &none { + panic("BUG: passed block with empty set") + } + return &s.root + } + + if next == &s.root { + // Special case: we need to create a new block that will become the root + // block.The old root block becomes the second block. + second := s.root + s.root = block{ + next: &second, + } + if second.next == &s.root { + s.root.prev = &second + } else { + s.root.prev = second.prev + second.next.prev = &second + second.prev = &s.root + } + return &s.root + } + if next == &none { + // Insert before root. + next = &s.root + } b := new(block) b.next = next b.prev = next.prev @@ -389,9 +503,13 @@ func (s *Sparse) insertBlockBefore(next *block) *block { // discardTail removes block b and all its successors from s. func (s *Sparse) discardTail(b *block) { - if b != &s.root { - b.prev.next = &s.root - s.root.prev = b.prev + if b != &none { + if b == &s.root { + s.Clear() + } else { + b.prev.next = &s.root + s.root.prev = b.prev + } } } @@ -401,16 +519,15 @@ func (s *Sparse) IntersectionWith(x *Sparse) { return } - xb := x.start() - sb := s.start() - for xb != &x.root && sb != &s.root { + xb := x.first() + sb := s.first() + for xb != &none && sb != &none { switch { case xb.offset < sb.offset: - xb = xb.next + xb = x.next(xb) case xb.offset > sb.offset: - sb = sb.next - s.removeBlock(sb.prev) + sb = s.removeBlock(sb) default: var sum word @@ -420,12 +537,12 @@ func (s *Sparse) IntersectionWith(x *Sparse) { sum |= r } if sum != 0 { - sb = sb.next + sb = s.next(sb) } else { // sb will be overwritten or removed } - xb = xb.next + xb = x.next(xb) } } @@ -446,20 +563,20 @@ func (s *Sparse) Intersection(x, y *Sparse) { return } - xb := x.start() - yb := y.start() - sb := s.start() - for xb != &x.root && yb != &y.root { + xb := x.first() + yb := y.first() + sb := s.first() + for xb != &none && yb != &none { switch { case xb.offset < yb.offset: - xb = xb.next + xb = x.next(xb) continue case xb.offset > yb.offset: - yb = yb.next + yb = y.next(yb) continue } - if sb == &s.root { + if sb == &none { sb = s.insertBlockBefore(sb) } sb.offset = xb.offset @@ -471,13 +588,13 @@ func (s *Sparse) Intersection(x, y *Sparse) { sum |= r } if sum != 0 { - sb = sb.next + sb = s.next(sb) } else { // sb will be overwritten or removed } - xb = xb.next - yb = yb.next + xb = x.next(xb) + yb = y.next(yb) } s.discardTail(sb) @@ -485,22 +602,22 @@ func (s *Sparse) Intersection(x, y *Sparse) { // Intersects reports whether s ∩ x ≠ ∅. func (s *Sparse) Intersects(x *Sparse) bool { - sb := s.start() - xb := x.start() - for sb != &s.root && xb != &x.root { + sb := s.first() + xb := x.first() + for sb != &none && xb != &none { switch { case xb.offset < sb.offset: - xb = xb.next + xb = x.next(xb) case xb.offset > sb.offset: - sb = sb.next + sb = s.next(sb) default: for i := range sb.bits { if sb.bits[i]&xb.bits[i] != 0 { return true } } - sb = sb.next - xb = xb.next + sb = s.next(sb) + xb = x.next(xb) } } return false @@ -513,26 +630,26 @@ func (s *Sparse) UnionWith(x *Sparse) bool { } var changed bool - xb := x.start() - sb := s.start() - for xb != &x.root { - if sb != &s.root && sb.offset == xb.offset { + xb := x.first() + sb := s.first() + for xb != &none { + if sb != &none && sb.offset == xb.offset { for i := range xb.bits { if sb.bits[i] != xb.bits[i] { sb.bits[i] |= xb.bits[i] changed = true } } - xb = xb.next - } else if sb == &s.root || sb.offset > xb.offset { + xb = x.next(xb) + } else if sb == &none || sb.offset > xb.offset { sb = s.insertBlockBefore(sb) sb.offset = xb.offset sb.bits = xb.bits changed = true - xb = xb.next + xb = x.next(xb) } - sb = sb.next + sb = s.next(sb) } return changed } @@ -551,33 +668,33 @@ func (s *Sparse) Union(x, y *Sparse) { return } - xb := x.start() - yb := y.start() - sb := s.start() - for xb != &x.root || yb != &y.root { - if sb == &s.root { + xb := x.first() + yb := y.first() + sb := s.first() + for xb != &none || yb != &none { + if sb == &none { sb = s.insertBlockBefore(sb) } switch { - case yb == &y.root || (xb != &x.root && xb.offset < yb.offset): + case yb == &none || (xb != &none && xb.offset < yb.offset): sb.offset = xb.offset sb.bits = xb.bits - xb = xb.next + xb = x.next(xb) - case xb == &x.root || (yb != &y.root && yb.offset < xb.offset): + case xb == &none || (yb != &none && yb.offset < xb.offset): sb.offset = yb.offset sb.bits = yb.bits - yb = yb.next + yb = y.next(yb) default: sb.offset = xb.offset for i := range xb.bits { sb.bits[i] = xb.bits[i] | yb.bits[i] } - xb = xb.next - yb = yb.next + xb = x.next(xb) + yb = y.next(yb) } - sb = sb.next + sb = s.next(sb) } s.discardTail(sb) @@ -590,15 +707,15 @@ func (s *Sparse) DifferenceWith(x *Sparse) { return } - xb := x.start() - sb := s.start() - for xb != &x.root && sb != &s.root { + xb := x.first() + sb := s.first() + for xb != &none && sb != &none { switch { case xb.offset > sb.offset: - sb = sb.next + sb = s.next(sb) case xb.offset < sb.offset: - xb = xb.next + xb = x.next(xb) default: var sum word @@ -607,12 +724,12 @@ func (s *Sparse) DifferenceWith(x *Sparse) { sb.bits[i] = r sum |= r } - sb = sb.next - xb = xb.next - if sum == 0 { - s.removeBlock(sb.prev) + sb = s.removeBlock(sb) + } else { + sb = s.next(sb) } + xb = x.next(xb) } } } @@ -633,27 +750,27 @@ func (s *Sparse) Difference(x, y *Sparse) { return } - xb := x.start() - yb := y.start() - sb := s.start() - for xb != &x.root && yb != &y.root { + xb := x.first() + yb := y.first() + sb := s.first() + for xb != &none && yb != &none { if xb.offset > yb.offset { - // y has block, x has none - yb = yb.next + // y has block, x has &none + yb = y.next(yb) continue } - if sb == &s.root { + if sb == &none { sb = s.insertBlockBefore(sb) } sb.offset = xb.offset switch { case xb.offset < yb.offset: - // x has block, y has none + // x has block, y has &none sb.bits = xb.bits - sb = sb.next + sb = s.next(sb) default: // x and y have corresponding blocks @@ -664,25 +781,25 @@ func (s *Sparse) Difference(x, y *Sparse) { sum |= r } if sum != 0 { - sb = sb.next + sb = s.next(sb) } else { // sb will be overwritten or removed } - yb = yb.next + yb = y.next(yb) } - xb = xb.next + xb = x.next(xb) } - for xb != &x.root { - if sb == &s.root { + for xb != &none { + if sb == &none { sb = s.insertBlockBefore(sb) } sb.offset = xb.offset sb.bits = xb.bits - sb = sb.next + sb = s.next(sb) - xb = xb.next + xb = x.next(xb) } s.discardTail(sb) @@ -695,17 +812,17 @@ func (s *Sparse) SymmetricDifferenceWith(x *Sparse) { return } - sb := s.start() - xb := x.start() - for xb != &x.root && sb != &s.root { + sb := s.first() + xb := x.first() + for xb != &none && sb != &none { switch { case sb.offset < xb.offset: - sb = sb.next + sb = s.next(sb) case xb.offset < sb.offset: nb := s.insertBlockBefore(sb) nb.offset = xb.offset nb.bits = xb.bits - xb = xb.next + xb = x.next(xb) default: var sum word for i := range sb.bits { @@ -713,20 +830,21 @@ func (s *Sparse) SymmetricDifferenceWith(x *Sparse) { sb.bits[i] = r sum |= r } - sb = sb.next - xb = xb.next if sum == 0 { - s.removeBlock(sb.prev) + sb = s.removeBlock(sb) + } else { + sb = s.next(sb) } + xb = x.next(xb) } } - for xb != &x.root { // append the tail of x to s + for xb != &none { // append the tail of x to s sb = s.insertBlockBefore(sb) sb.offset = xb.offset sb.bits = xb.bits - sb = sb.next - xb = xb.next + sb = s.next(sb) + xb = x.next(xb) } } @@ -744,24 +862,24 @@ func (s *Sparse) SymmetricDifference(x, y *Sparse) { return } - sb := s.start() - xb := x.start() - yb := y.start() - for xb != &x.root && yb != &y.root { - if sb == &s.root { + sb := s.first() + xb := x.first() + yb := y.first() + for xb != &none && yb != &none { + if sb == &none { sb = s.insertBlockBefore(sb) } switch { case yb.offset < xb.offset: sb.offset = yb.offset sb.bits = yb.bits - sb = sb.next - yb = yb.next + sb = s.next(sb) + yb = y.next(yb) case xb.offset < yb.offset: sb.offset = xb.offset sb.bits = xb.bits - sb = sb.next - xb = xb.next + sb = s.next(sb) + xb = x.next(xb) default: var sum word for i := range sb.bits { @@ -771,31 +889,31 @@ func (s *Sparse) SymmetricDifference(x, y *Sparse) { } if sum != 0 { sb.offset = xb.offset - sb = sb.next + sb = s.next(sb) } - xb = xb.next - yb = yb.next + xb = x.next(xb) + yb = y.next(yb) } } - for xb != &x.root { // append the tail of x to s - if sb == &s.root { + for xb != &none { // append the tail of x to s + if sb == &none { sb = s.insertBlockBefore(sb) } sb.offset = xb.offset sb.bits = xb.bits - sb = sb.next - xb = xb.next + sb = s.next(sb) + xb = x.next(xb) } - for yb != &y.root { // append the tail of y to s - if sb == &s.root { + for yb != &none { // append the tail of y to s + if sb == &none { sb = s.insertBlockBefore(sb) } sb.offset = yb.offset sb.bits = yb.bits - sb = sb.next - yb = yb.next + sb = s.next(sb) + yb = y.next(yb) } s.discardTail(sb) @@ -807,22 +925,22 @@ func (s *Sparse) SubsetOf(x *Sparse) bool { return true } - sb := s.start() - xb := x.start() - for sb != &s.root { + sb := s.first() + xb := x.first() + for sb != &none { switch { - case xb == &x.root || xb.offset > sb.offset: + case xb == &none || xb.offset > sb.offset: return false case xb.offset < sb.offset: - xb = xb.next + xb = x.next(xb) default: for i := range sb.bits { if sb.bits[i]&^xb.bits[i] != 0 { return false } } - sb = sb.next - xb = xb.next + sb = s.next(sb) + xb = x.next(xb) } } return true @@ -833,13 +951,13 @@ func (s *Sparse) Equals(t *Sparse) bool { if s == t { return true } - sb := s.start() - tb := t.start() + sb := s.first() + tb := t.first() for { switch { - case sb == &s.root && tb == &t.root: + case sb == &none && tb == &none: return true - case sb == &s.root || tb == &t.root: + case sb == &none || tb == &none: return false case sb.offset != tb.offset: return false @@ -847,8 +965,8 @@ func (s *Sparse) Equals(t *Sparse) bool { return false } - sb = sb.next - tb = tb.next + sb = s.next(sb) + tb = t.next(tb) } } @@ -913,7 +1031,7 @@ func (s *Sparse) BitString() string { // func (s *Sparse) GoString() string { var buf bytes.Buffer - for b := s.start(); b != &s.root; b = b.next { + for b := s.first(); b != &none; b = s.next(b) { fmt.Fprintf(&buf, "block %p {offset=%d next=%p prev=%p", b, b.offset, b.next, b.prev) for _, w := range b.bits { @@ -937,13 +1055,18 @@ func (s *Sparse) AppendTo(slice []int) []int { // check returns an error if the representation invariants of s are violated. func (s *Sparse) check() error { - if !s.root.empty() { - return fmt.Errorf("non-empty root block") + s.init() + if s.root.empty() { + // An empty set must have only the root block with offset MaxInt. + if s.root.next != &s.root { + return fmt.Errorf("multiple blocks with empty root block") + } + if s.root.offset != MaxInt { + return fmt.Errorf("empty set has offset %d, should be MaxInt", s.root.offset) + } + return nil } - if s.root.offset != 0 { - return fmt.Errorf("root block has non-zero offset %d", s.root.offset) - } - for b := s.start(); b != &s.root; b = b.next { + for b := s.first(); ; b = s.next(b) { if b.offset%bitsPerBlock != 0 { return fmt.Errorf("bad offset modulo: %d", b.offset) } @@ -956,11 +1079,12 @@ func (s *Sparse) check() error { if b.next.prev != b { return fmt.Errorf("bad next.prev link") } - if b.prev != &s.root { - if b.offset <= b.prev.offset { - return fmt.Errorf("bad offset order: b.offset=%d, prev.offset=%d", - b.offset, b.prev.offset) - } + if b.next == &s.root { + break + } + if b.offset >= b.next.offset { + return fmt.Errorf("bad offset order: b.offset=%d, b.next.offset=%d", + b.offset, b.next.offset) } } return nil diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/go/ast/astutil/imports.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/go/ast/astutil/imports.go index 4a77c7828..83f196cd5 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/go/ast/astutil/imports.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/go/ast/astutil/imports.go @@ -49,6 +49,8 @@ func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added lastImport = -1 // index in f.Decls of the file's final import decl impDecl *ast.GenDecl // import decl containing the best match impIndex = -1 // spec index in impDecl containing the best match + + isThirdPartyPath = isThirdParty(ipath) ) for i, decl := range f.Decls { gen, ok := decl.(*ast.GenDecl) @@ -65,15 +67,27 @@ func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added impDecl = gen } - // Compute longest shared prefix with imports in this group. + // Compute longest shared prefix with imports in this group and find best + // matched import spec. + // 1. Always prefer import spec with longest shared prefix. + // 2. While match length is 0, + // - for stdlib package: prefer first import spec. + // - for third party package: prefer first third party import spec. + // We cannot use last import spec as best match for third party package + // because grouped imports are usually placed last by goimports -local + // flag. + // See issue #19190. + seenAnyThirdParty := false for j, spec := range gen.Specs { impspec := spec.(*ast.ImportSpec) - n := matchLen(importPath(impspec), ipath) - if n > bestMatch { + p := importPath(impspec) + n := matchLen(p, ipath) + if n > bestMatch || (bestMatch == 0 && !seenAnyThirdParty && isThirdPartyPath) { bestMatch = n impDecl = gen impIndex = j } + seenAnyThirdParty = seenAnyThirdParty || isThirdParty(p) } } } @@ -175,6 +189,12 @@ func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added return true } +func isThirdParty(importPath string) bool { + // Third party package import path usually contains "." (".com", ".org", ...) + // This logic is taken from golang.org/x/tools/imports package. + return strings.Contains(importPath, ".") +} + // DeleteImport deletes the import path from the file f, if present. func DeleteImport(fset *token.FileSet, f *ast.File, path string) (deleted bool) { return DeleteNamedImport(fset, f, "", path) diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/fastwalk_dirent_ino.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/fastwalk_dirent_ino.go index ee85bc4dd..9fc6de038 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/fastwalk_dirent_ino.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/fastwalk_dirent_ino.go @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build linux,!appengine darwin +// +build linux darwin +// +build !appengine package imports diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/fastwalk_unix.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/fastwalk_unix.go index 5854233db..db6ee8daf 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/fastwalk_unix.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/fastwalk_unix.go @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build linux,!appengine darwin freebsd openbsd netbsd +// +build linux darwin freebsd openbsd netbsd +// +build !appengine package imports diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/fix.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/fix.go index ac7f4b00d..e24608497 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/fix.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/fix.go @@ -776,7 +776,7 @@ func findImportGoPath(pkgName string, symbols map[string]bool, filename string) sort.Sort(byImportPathShortLength(candidates)) if Debug { for i, pkg := range candidates { - log.Printf("%s candidate %d/%d: %v", pkgName, i+1, len(candidates), pkg.importPathShort) + log.Printf("%s candidate %d/%d: %v in %v", pkgName, i+1, len(candidates), pkg.importPathShort, pkg.dir) } } diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/mkstdlib.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/mkstdlib.go index 5602244a7..6e6e1d6dc 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/mkstdlib.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/mkstdlib.go @@ -53,6 +53,7 @@ func main() { mustOpen(api("go1.6.txt")), mustOpen(api("go1.7.txt")), mustOpen(api("go1.8.txt")), + mustOpen(api("go1.9.txt")), ) sc := bufio.NewScanner(f) fullImport := map[string]string{} // "zip.NewReader" => "archive/zip" diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/zstdlib.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/zstdlib.go index 5b66a6cd4..974c2d040 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/zstdlib.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/golang.org/x/tools/imports/zstdlib.go @@ -23,6 +23,8 @@ var stdlib = map[string]string{ "asn1.Enumerated": "encoding/asn1", "asn1.Flag": "encoding/asn1", "asn1.Marshal": "encoding/asn1", + "asn1.NullBytes": "encoding/asn1", + "asn1.NullRawValue": "encoding/asn1", "asn1.ObjectIdentifier": "encoding/asn1", "asn1.RawContent": "encoding/asn1", "asn1.RawValue": "encoding/asn1", @@ -35,6 +37,7 @@ var stdlib = map[string]string{ "asn1.TagGeneralizedTime": "encoding/asn1", "asn1.TagIA5String": "encoding/asn1", "asn1.TagInteger": "encoding/asn1", + "asn1.TagNull": "encoding/asn1", "asn1.TagOID": "encoding/asn1", "asn1.TagOctetString": "encoding/asn1", "asn1.TagPrintableString": "encoding/asn1", @@ -177,7 +180,9 @@ var stdlib = map[string]string{ "base32.NewDecoder": "encoding/base32", "base32.NewEncoder": "encoding/base32", "base32.NewEncoding": "encoding/base32", + "base32.NoPadding": "encoding/base32", "base32.StdEncoding": "encoding/base32", + "base32.StdPadding": "encoding/base32", "base64.CorruptInputError": "encoding/base64", "base64.Encoding": "encoding/base64", "base64.NewDecoder": "encoding/base64", @@ -229,6 +234,41 @@ var stdlib = map[string]string{ "binary.Uvarint": "encoding/binary", "binary.Varint": "encoding/binary", "binary.Write": "encoding/binary", + "bits.LeadingZeros": "math/bits", + "bits.LeadingZeros16": "math/bits", + "bits.LeadingZeros32": "math/bits", + "bits.LeadingZeros64": "math/bits", + "bits.LeadingZeros8": "math/bits", + "bits.Len": "math/bits", + "bits.Len16": "math/bits", + "bits.Len32": "math/bits", + "bits.Len64": "math/bits", + "bits.Len8": "math/bits", + "bits.OnesCount": "math/bits", + "bits.OnesCount16": "math/bits", + "bits.OnesCount32": "math/bits", + "bits.OnesCount64": "math/bits", + "bits.OnesCount8": "math/bits", + "bits.Reverse": "math/bits", + "bits.Reverse16": "math/bits", + "bits.Reverse32": "math/bits", + "bits.Reverse64": "math/bits", + "bits.Reverse8": "math/bits", + "bits.ReverseBytes": "math/bits", + "bits.ReverseBytes16": "math/bits", + "bits.ReverseBytes32": "math/bits", + "bits.ReverseBytes64": "math/bits", + "bits.RotateLeft": "math/bits", + "bits.RotateLeft16": "math/bits", + "bits.RotateLeft32": "math/bits", + "bits.RotateLeft64": "math/bits", + "bits.RotateLeft8": "math/bits", + "bits.TrailingZeros": "math/bits", + "bits.TrailingZeros16": "math/bits", + "bits.TrailingZeros32": "math/bits", + "bits.TrailingZeros64": "math/bits", + "bits.TrailingZeros8": "math/bits", + "bits.UintSize": "math/bits", "bufio.ErrAdvanceTooFar": "bufio", "bufio.ErrBufferFull": "bufio", "bufio.ErrFinalToken": "bufio", @@ -471,6 +511,10 @@ var stdlib = map[string]string{ "crc64.Size": "hash/crc64", "crc64.Table": "hash/crc64", "crc64.Update": "hash/crc64", + "crypto.BLAKE2b_256": "crypto", + "crypto.BLAKE2b_384": "crypto", + "crypto.BLAKE2b_512": "crypto", + "crypto.BLAKE2s_256": "crypto", "crypto.Decrypter": "crypto", "crypto.DecrypterOpts": "crypto", "crypto.Hash": "crypto", @@ -552,6 +596,7 @@ var stdlib = map[string]string{ "driver.DefaultParameterConverter": "database/sql/driver", "driver.Driver": "database/sql/driver", "driver.ErrBadConn": "database/sql/driver", + "driver.ErrRemoveArgument": "database/sql/driver", "driver.ErrSkip": "database/sql/driver", "driver.Execer": "database/sql/driver", "driver.ExecerContext": "database/sql/driver", @@ -560,6 +605,7 @@ var stdlib = map[string]string{ "driver.IsValue": "database/sql/driver", "driver.IsolationLevel": "database/sql/driver", "driver.NamedValue": "database/sql/driver", + "driver.NamedValueChecker": "database/sql/driver", "driver.NotNull": "database/sql/driver", "driver.Null": "database/sql/driver", "driver.Pinger": "database/sql/driver", @@ -1690,6 +1736,7 @@ var stdlib = map[string]string{ "expvar.Var": "expvar", "fcgi.ErrConnClosed": "net/http/fcgi", "fcgi.ErrRequestAborted": "net/http/fcgi", + "fcgi.ProcessEnv": "net/http/fcgi", "fcgi.Serve": "net/http/fcgi", "filepath.Abs": "path/filepath", "filepath.Base": "path/filepath", @@ -1796,6 +1843,8 @@ var stdlib = map[string]string{ "fmt.Sscanln": "fmt", "fmt.State": "fmt", "fmt.Stringer": "fmt", + "fnv.New128": "hash/fnv", + "fnv.New128a": "hash/fnv", "fnv.New32": "hash/fnv", "fnv.New32a": "hash/fnv", "fnv.New64": "hash/fnv", @@ -1954,6 +2003,7 @@ var stdlib = map[string]string{ "http.ServeContent": "net/http", "http.ServeFile": "net/http", "http.ServeMux": "net/http", + "http.ServeTLS": "net/http", "http.Server": "net/http", "http.ServerContextKey": "net/http", "http.SetCookie": "net/http", @@ -2205,6 +2255,7 @@ var stdlib = map[string]string{ "json.Unmarshaler": "encoding/json", "json.UnsupportedTypeError": "encoding/json", "json.UnsupportedValueError": "encoding/json", + "json.Valid": "encoding/json", "jsonrpc.Dial": "net/rpc/jsonrpc", "jsonrpc.NewClient": "net/rpc/jsonrpc", "jsonrpc.NewClientCodec": "net/rpc/jsonrpc", @@ -2402,6 +2453,7 @@ var stdlib = map[string]string{ "md5.Sum": "crypto/md5", "mime.AddExtensionType": "mime", "mime.BEncoding": "mime", + "mime.ErrInvalidMediaParameter": "mime", "mime.ExtensionsByType": "mime", "mime.FormatMediaType": "mime", "mime.ParseMediaType": "mime", @@ -2409,6 +2461,7 @@ var stdlib = map[string]string{ "mime.TypeByExtension": "mime", "mime.WordDecoder": "mime", "mime.WordEncoder": "mime", + "multipart.ErrMessageTooLarge": "mime/multipart", "multipart.File": "mime/multipart", "multipart.FileHeader": "mime/multipart", "multipart.Form": "mime/multipart", @@ -2753,20 +2806,29 @@ var stdlib = map[string]string{ "png.DefaultCompression": "image/png", "png.Encode": "image/png", "png.Encoder": "image/png", + "png.EncoderBuffer": "image/png", + "png.EncoderBufferPool": "image/png", "png.FormatError": "image/png", "png.NoCompression": "image/png", "png.UnsupportedError": "image/png", "pprof.Cmdline": "net/http/pprof", + "pprof.Do": "runtime/pprof", + "pprof.ForLabels": "runtime/pprof", "pprof.Handler": "net/http/pprof", "pprof.Index": "net/http/pprof", + "pprof.Label": "runtime/pprof", + "pprof.LabelSet": "runtime/pprof", + "pprof.Labels": "runtime/pprof", "pprof.Lookup": "runtime/pprof", "pprof.NewProfile": "runtime/pprof", // "pprof.Profile" is ambiguous "pprof.Profiles": "runtime/pprof", + "pprof.SetGoroutineLabels": "runtime/pprof", "pprof.StartCPUProfile": "runtime/pprof", "pprof.StopCPUProfile": "runtime/pprof", "pprof.Symbol": "net/http/pprof", "pprof.Trace": "net/http/pprof", + "pprof.WithLabels": "runtime/pprof", "pprof.WriteHeapProfile": "runtime/pprof", "printer.CommentedNode": "go/printer", "printer.Config": "go/printer", @@ -2844,6 +2906,7 @@ var stdlib = map[string]string{ "reflect.MakeChan": "reflect", "reflect.MakeFunc": "reflect", "reflect.MakeMap": "reflect", + "reflect.MakeMapWithSize": "reflect", "reflect.MakeSlice": "reflect", "reflect.Map": "reflect", "reflect.MapOf": "reflect", @@ -3071,9 +3134,11 @@ var stdlib = map[string]string{ "sort.Strings": "sort", "sort.StringsAreSorted": "sort", "sql.ColumnType": "database/sql", + "sql.Conn": "database/sql", "sql.DB": "database/sql", "sql.DBStats": "database/sql", "sql.Drivers": "database/sql", + "sql.ErrConnDone": "database/sql", "sql.ErrNoRows": "database/sql", "sql.ErrTxDone": "database/sql", "sql.IsolationLevel": "database/sql", @@ -3092,6 +3157,7 @@ var stdlib = map[string]string{ "sql.NullInt64": "database/sql", "sql.NullString": "database/sql", "sql.Open": "database/sql", + "sql.Out": "database/sql", "sql.RawBytes": "database/sql", "sql.Register": "database/sql", "sql.Result": "database/sql", @@ -3193,6 +3259,7 @@ var stdlib = map[string]string{ "suffixarray.New": "index/suffixarray", "sync.Cond": "sync", "sync.Locker": "sync", + "sync.Map": "sync", "sync.Mutex": "sync", "sync.NewCond": "sync", "sync.Once": "sync", @@ -3772,6 +3839,7 @@ var stdlib = map[string]string{ "syscall.Cmsghdr": "syscall", "syscall.CommandLineToArgv": "syscall", "syscall.ComputerName": "syscall", + "syscall.Conn": "syscall", "syscall.Connect": "syscall", "syscall.ConnectEx": "syscall", "syscall.ConvertSidToStringSid": "syscall", @@ -6581,6 +6649,7 @@ var stdlib = map[string]string{ "syscall.RUSAGE_SELF": "syscall", "syscall.RUSAGE_THREAD": "syscall", "syscall.Radvisory_t": "syscall", + "syscall.RawConn": "syscall", "syscall.RawSockaddr": "syscall", "syscall.RawSockaddrAny": "syscall", "syscall.RawSockaddrDatalink": "syscall", @@ -8357,6 +8426,7 @@ var stdlib = map[string]string{ "syscall.WSADESCRIPTION_LEN": "syscall", "syscall.WSAData": "syscall", "syscall.WSAEACCES": "syscall", + "syscall.WSAECONNABORTED": "syscall", "syscall.WSAECONNRESET": "syscall", "syscall.WSAEnumProtocols": "syscall", "syscall.WSAID_CONNECTEX": "syscall", @@ -8478,6 +8548,7 @@ var stdlib = map[string]string{ "template.ErrOutputContext": "html/template", "template.ErrPartialCharset": "html/template", "template.ErrPartialEscape": "html/template", + "template.ErrPredefinedEscaper": "html/template", "template.ErrRangeLoopReentry": "html/template", "template.ErrSlashAmbig": "html/template", "template.Error": "html/template", @@ -8871,6 +8942,7 @@ var stdlib = map[string]string{ "types.SendRecv": "go/types", "types.Signature": "go/types", "types.Sizes": "go/types", + "types.SizesFor": "go/types", "types.Slice": "go/types", "types.StdSizes": "go/types", "types.String": "go/types", diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/honnef.co/go/tools/functions/functions.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/honnef.co/go/tools/functions/functions.go index 477f8a5a3..9213600c5 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/honnef.co/go/tools/functions/functions.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/honnef.co/go/tools/functions/functions.go @@ -11,6 +11,14 @@ import ( ) var stdlibDescs = map[string]Description{ + "errors.New": Description{Pure: true}, + + "fmt.Errorf": Description{Pure: true}, + "fmt.Sprintf": Description{Pure: true}, + "fmt.Sprint": Description{Pure: true}, + + "sort.Reverse": Description{Pure: true}, + "strings.Map": Description{Pure: true}, "strings.Repeat": Description{Pure: true}, "strings.Replace": Description{Pure: true}, diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/honnef.co/go/tools/simple/lint.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/honnef.co/go/tools/simple/lint.go index 9ea35d929..bff623415 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/honnef.co/go/tools/simple/lint.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/honnef.co/go/tools/simple/lint.go @@ -6,9 +6,7 @@ import ( "go/constant" "go/token" "go/types" - "math" "reflect" - "strconv" "strings" "honnef.co/go/tools/internal/sharedcheck" @@ -42,7 +40,7 @@ func (c *Checker) Funcs() map[string]lint.Func { "S1002": c.LintIfBoolCmp, "S1003": c.LintStringsContains, "S1004": c.LintBytesCompare, - "S1005": c.LintRanges, + "S1005": c.LintUnnecessaryBlank, "S1006": c.LintForTrue, "S1007": c.LintRegexpRaw, "S1008": c.LintIfReturn, @@ -51,22 +49,24 @@ func (c *Checker) Funcs() map[string]lint.Func { "S1011": c.LintLoopAppend, "S1012": c.LintTimeSince, "S1013": c.LintSimplerReturn, - "S1014": c.LintReceiveIntoBlank, - "S1015": c.LintFormatInt, + "S1014": nil, + "S1015": nil, "S1016": c.LintSimplerStructConversion, "S1017": c.LintTrim, "S1018": c.LintLoopSlide, "S1019": c.LintMakeLenCap, "S1020": c.LintAssertNotNil, "S1021": c.LintDeclareAssign, - "S1022": c.LintBlankOK, + "S1022": nil, "S1023": c.LintRedundantBreak, "S1024": c.LintTimeUntil, "S1025": c.LintRedundantSprintf, "S1026": c.LintStringCopy, - "S1027": c.LintRedundantReturn, + "S1027": nil, "S1028": c.LintErrorsNewSprintf, "S1029": c.LintRangeStringRunes, + "S1030": c.LintBytesBufferConversions, + "S1031": c.LintNilCheckAroundRange, } } @@ -247,6 +247,36 @@ func (c *Checker) LintIfBoolCmp(j *lint.Job) { } } +func (c *Checker) LintBytesBufferConversions(j *lint.Job) { + fn := func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok || len(call.Args) != 1 { + return true + } + + argCall, ok := call.Args[0].(*ast.CallExpr) + if !ok { + return true + } + sel, ok := argCall.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + + typ := j.Program.Info.TypeOf(call.Fun) + if typ == types.Universe.Lookup("string").Type() && j.IsCallToAST(call.Args[0], "(*bytes.Buffer).Bytes") { + j.Errorf(call, "should use %v.String() instead of %v", j.Render(sel.X), j.Render(call)) + } else if typ, ok := typ.(*types.Slice); ok && typ.Elem() == types.Universe.Lookup("byte").Type() && j.IsCallToAST(call.Args[0], "(*bytes.Buffer).String") { + j.Errorf(call, "should use %v.Bytes() instead of %v", j.Render(sel.X), j.Render(call)) + } + + return true + } + for _, f := range c.filterGenerated(j.Program.Files) { + ast.Inspect(f, fn) + } +} + func (c *Checker) LintStringsContains(j *lint.Job) { // map of value to token to bool value allowed := map[int64]map[token.Token]bool{ @@ -352,23 +382,6 @@ func (c *Checker) LintBytesCompare(j *lint.Job) { } } -func (c *Checker) LintRanges(j *lint.Job) { - fn := func(node ast.Node) bool { - rs, ok := node.(*ast.RangeStmt) - if !ok { - return true - } - if lint.IsBlank(rs.Key) && (rs.Value == nil || lint.IsBlank(rs.Value)) { - j.Errorf(rs.Key, "should omit values from range; this loop is equivalent to `for range ...`") - } - - return true - } - for _, f := range c.filterGenerated(j.Program.Files) { - ast.Inspect(f, fn) - } -} - func (c *Checker) LintForTrue(j *lint.Job) { fn := func(node ast.Node) bool { loop, ok := node.(*ast.ForStmt) @@ -941,14 +954,44 @@ func (c *Checker) LintSimplerReturn(j *lint.Job) { } } -func (c *Checker) LintReceiveIntoBlank(j *lint.Job) { - fn := func(node ast.Node) bool { +func (c *Checker) LintUnnecessaryBlank(j *lint.Job) { + fn1 := func(node ast.Node) { + assign, ok := node.(*ast.AssignStmt) + if !ok { + return + } + if len(assign.Lhs) != 2 || len(assign.Rhs) != 1 { + return + } + if !lint.IsBlank(assign.Lhs[1]) { + return + } + switch rhs := assign.Rhs[0].(type) { + case *ast.IndexExpr: + // The type-checker should make sure that it's a map, but + // let's be safe. + if _, ok := j.Program.Info.TypeOf(rhs.X).Underlying().(*types.Map); !ok { + return + } + case *ast.UnaryExpr: + if rhs.Op != token.ARROW { + return + } + default: + return + } + cp := *assign + cp.Lhs = cp.Lhs[0:1] + j.Errorf(assign, "should write %s instead of %s", j.Render(&cp), j.Render(assign)) + } + + fn2 := func(node ast.Node) { stmt, ok := node.(*ast.AssignStmt) if !ok { - return true + return } if len(stmt.Lhs) != len(stmt.Rhs) { - return true + return } for i, lh := range stmt.Lhs { rh := stmt.Rhs[i] @@ -964,101 +1007,22 @@ func (c *Checker) LintReceiveIntoBlank(j *lint.Job) { } j.Errorf(lh, "'_ = <-ch' can be simplified to '<-ch'") } - return true } - for _, f := range c.filterGenerated(j.Program.Files) { - ast.Inspect(f, fn) - } -} -func (c *Checker) LintFormatInt(j *lint.Job) { - checkBasic := func(v ast.Expr) bool { - typ, ok := j.Program.Info.TypeOf(v).(*types.Basic) + fn3 := func(node ast.Node) { + rs, ok := node.(*ast.RangeStmt) if !ok { - return false + return } - return typ.Kind() == types.Int - } - checkConst := func(v *ast.Ident) bool { - c, ok := j.Program.Info.ObjectOf(v).(*types.Const) - if !ok { - return false + if lint.IsBlank(rs.Key) && (rs.Value == nil || lint.IsBlank(rs.Value)) { + j.Errorf(rs.Key, "should omit values from range; this loop is equivalent to `for range ...`") } - if c.Val().Kind() != constant.Int { - return false - } - i, _ := constant.Int64Val(c.Val()) - return i <= math.MaxInt32 - } - checkConstStrict := func(v *ast.Ident) bool { - if !checkConst(v) { - return false - } - basic, ok := j.Program.Info.ObjectOf(v).(*types.Const).Type().(*types.Basic) - return ok && basic.Kind() == types.UntypedInt } fn := func(node ast.Node) bool { - call, ok := node.(*ast.CallExpr) - if !ok { - return true - } - if !j.IsCallToAST(call, "strconv.FormatInt") { - return true - } - if len(call.Args) != 2 { - return true - } - if lit, ok := call.Args[1].(*ast.BasicLit); !ok || lit.Value != "10" { - return true - } - - matches := false - switch v := call.Args[0].(type) { - case *ast.CallExpr: - if len(v.Args) != 1 { - return true - } - ident, ok := v.Fun.(*ast.Ident) - if !ok { - return true - } - obj, ok := j.Program.Info.ObjectOf(ident).(*types.TypeName) - if !ok || obj.Parent() != types.Universe || obj.Name() != "int64" { - return true - } - - switch vv := v.Args[0].(type) { - case *ast.BasicLit: - i, _ := strconv.ParseInt(vv.Value, 10, 64) - if i <= math.MaxInt32 { - matches = true - } - case *ast.Ident: - if checkConst(vv) || checkBasic(v.Args[0]) { - matches = true - } - default: - if checkBasic(v.Args[0]) { - matches = true - } - } - case *ast.BasicLit: - if v.Kind != token.INT { - return true - } - i, _ := strconv.ParseInt(v.Value, 10, 64) - if i <= math.MaxInt32 { - matches = true - } - case *ast.Ident: - if checkConstStrict(v) { - matches = true - } - } - if matches { - j.Errorf(call, "should use strconv.Itoa instead of strconv.FormatInt") - } + fn1(node) + fn2(node) + fn3(node) return true } for _, f := range c.filterGenerated(j.Program.Files) { @@ -1067,23 +1031,34 @@ func (c *Checker) LintFormatInt(j *lint.Job) { } func (c *Checker) LintSimplerStructConversion(j *lint.Job) { + var skip ast.Node fn := func(node ast.Node) bool { + // Do not suggest type conversion between pointers + if unary, ok := node.(*ast.UnaryExpr); ok && unary.Op == token.AND { + if lit, ok := unary.X.(*ast.CompositeLit); ok { + skip = lit + } + return true + } + + if node == skip { + return true + } + lit, ok := node.(*ast.CompositeLit) if !ok { return true } - typ1 := j.Program.Info.TypeOf(lit.Type) + typ1, _ := j.Program.Info.TypeOf(lit.Type).(*types.Named) if typ1 == nil { return true } - // FIXME support pointer to struct s1, ok := typ1.Underlying().(*types.Struct) if !ok { return true } - n := s1.NumFields() - var typ2 types.Type + var typ2 *types.Named var ident *ast.Ident getSelType := func(expr ast.Expr) (types.Type, *ast.Ident, bool) { sel, ok := expr.(*ast.SelectorExpr) @@ -1100,8 +1075,10 @@ func (c *Checker) LintSimplerStructConversion(j *lint.Job) { if len(lit.Elts) == 0 { return true } + if s1.NumFields() != len(lit.Elts) { + return true + } for i, elt := range lit.Elts { - n-- var t types.Type var id *ast.Ident var ok bool @@ -1129,21 +1106,27 @@ func (c *Checker) LintSimplerStructConversion(j *lint.Job) { if !ok { return true } - if typ2 != nil && typ2 != t { - return true - } + // All fields must be initialized from the same object if ident != nil && ident.Obj != id.Obj { return true } - typ2 = t + typ2, _ = t.(*types.Named) + if typ2 == nil { + return true + } ident = id } - if n != 0 { + if typ2 == nil { return true } - if typ2 == nil { + if typ1.Obj().Pkg() != typ2.Obj().Pkg() { + // Do not suggest type conversions between different + // packages. Types in different packages might only match + // by coincidence. Furthermore, if the dependency ever + // adds more fields to its type, it could break the code + // that relies on the type conversion to work. return true } @@ -1157,7 +1140,8 @@ func (c *Checker) LintSimplerStructConversion(j *lint.Job) { if !structsIdentical(s1, s2) { return true } - j.Errorf(node, "should use type conversion instead of struct literal") + j.Errorf(node, "should convert %s (type %s) to %s instead of using struct literal", + ident.Name, typ2.Obj().Name(), typ1.Obj().Name()) return true } for _, f := range c.filterGenerated(j.Program.Files) { @@ -1598,56 +1582,52 @@ func (c *Checker) LintDeclareAssign(j *lint.Job) { } } -func (c *Checker) LintBlankOK(j *lint.Job) { - fn := func(node ast.Node) bool { - assign, ok := node.(*ast.AssignStmt) - if !ok { - return true - } - if len(assign.Lhs) != 2 || len(assign.Rhs) != 1 { - return true - } - if !lint.IsBlank(assign.Lhs[1]) { - return true - } - switch rhs := assign.Rhs[0].(type) { - case *ast.IndexExpr: - // The type-checker should make sure that it's a map, but - // let's be safe. - if _, ok := j.Program.Info.TypeOf(rhs.X).Underlying().(*types.Map); !ok { - return true - } - case *ast.UnaryExpr: - if rhs.Op != token.ARROW { - return true - } - default: - return true - } - cp := *assign - cp.Lhs = cp.Lhs[0:1] - j.Errorf(assign, "should write %s instead of %s", j.Render(&cp), j.Render(assign)) - return true - } - for _, f := range c.filterGenerated(j.Program.Files) { - ast.Inspect(f, fn) - } -} - func (c *Checker) LintRedundantBreak(j *lint.Job) { - fn := func(node ast.Node) bool { + fn1 := func(node ast.Node) { clause, ok := node.(*ast.CaseClause) if !ok { - return true + return } if len(clause.Body) < 2 { - return true + return } branch, ok := clause.Body[len(clause.Body)-1].(*ast.BranchStmt) if !ok || branch.Tok != token.BREAK || branch.Label != nil { - return true + return } j.Errorf(branch, "redundant break statement") + return + } + fn2 := func(node ast.Node) { + var ret *ast.FieldList + var body *ast.BlockStmt + switch x := node.(type) { + case *ast.FuncDecl: + ret = x.Type.Results + body = x.Body + case *ast.FuncLit: + ret = x.Type.Results + body = x.Body + default: + return + } + // if the func has results, a return can't be redundant. + // similarly, if there are no statements, there can be + // no return. + if ret != nil || body == nil || len(body.List) < 1 { + return + } + rst, ok := body.List[len(body.List)-1].(*ast.ReturnStmt) + if !ok { + return + } + // we don't need to check rst.Results as we already + // checked x.Type.Results to be nil. + j.Errorf(rst, "redundant return statement") + } + fn := func(node ast.Node) bool { + fn1(node) + fn2(node) return true } for _, f := range c.filterGenerated(j.Program.Files) { @@ -1797,40 +1777,6 @@ func (c *Checker) LintStringCopy(j *lint.Job) { } } -func (c *Checker) LintRedundantReturn(j *lint.Job) { - fn := func(node ast.Node) bool { - var ret *ast.FieldList - var body *ast.BlockStmt - switch x := node.(type) { - case *ast.FuncDecl: - ret = x.Type.Results - body = x.Body - case *ast.FuncLit: - ret = x.Type.Results - body = x.Body - default: - return true - } - // if the func has results, a return can't be redundant. - // similarly, if there are no statements, there can be - // no return. - if ret != nil || body == nil || len(body.List) < 1 { - return true - } - rst, ok := body.List[len(body.List)-1].(*ast.ReturnStmt) - if !ok { - return true - } - // we don't need to check rst.Results as we already - // checked x.Type.Results to be nil. - j.Errorf(rst, "redundant return statement") - return true - } - for _, f := range c.filterGenerated(j.Program.Files) { - ast.Inspect(f, fn) - } -} - func (c *Checker) LintErrorsNewSprintf(j *lint.Job) { fn := func(node ast.Node) bool { if !j.IsCallToAST(node, "errors.New") { @@ -1851,3 +1797,45 @@ func (c *Checker) LintErrorsNewSprintf(j *lint.Job) { func (c *Checker) LintRangeStringRunes(j *lint.Job) { sharedcheck.CheckRangeStringRunes(c.nodeFns, j) } + +func (c *Checker) LintNilCheckAroundRange(j *lint.Job) { + fn := func(node ast.Node) bool { + ifstmt, ok := node.(*ast.IfStmt) + if !ok { + return true + } + + cond, ok := ifstmt.Cond.(*ast.BinaryExpr) + if !ok { + return true + } + + if cond.Op != token.NEQ || !j.IsNil(cond.Y) || len(ifstmt.Body.List) != 1 { + return true + } + + loop, ok := ifstmt.Body.List[0].(*ast.RangeStmt) + if !ok { + return true + } + ifXIdent, ok := cond.X.(*ast.Ident) + if !ok { + return true + } + rangeXIdent, ok := loop.X.(*ast.Ident) + if !ok { + return true + } + if ifXIdent.Obj != rangeXIdent.Obj { + return true + } + switch j.Program.Info.TypeOf(rangeXIdent).(type) { + case *types.Slice, *types.Map: + j.Errorf(node, "unnecessary nil check around range") + } + return true + } + for _, f := range c.filterGenerated(j.Program.Files) { + ast.Inspect(f, fn) + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/honnef.co/go/tools/staticcheck/lint.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/honnef.co/go/tools/staticcheck/lint.go index c9ab9a9cc..4ab90386e 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/honnef.co/go/tools/staticcheck/lint.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/honnef.co/go/tools/staticcheck/lint.go @@ -10,6 +10,7 @@ import ( "go/types" htmltemplate "html/template" "net/http" + "regexp" "strconv" "strings" "sync" @@ -232,7 +233,7 @@ func (c *Checker) Funcs() map[string]lint.Func { "SA1019": c.CheckDeprecated, "SA1020": c.callChecker(checkListenAddressRules), "SA1021": c.callChecker(checkBytesEqualIPRules), - "SA1022": c.CheckFlagUsage, + "SA1022": nil, "SA1023": c.CheckWriterBufferModified, "SA1024": c.callChecker(checkUniqueCutsetRules), @@ -249,7 +250,7 @@ func (c *Checker) Funcs() map[string]lint.Func { "SA4002": c.CheckDiffSizeComparison, "SA4003": c.CheckUnsignedComparison, "SA4004": c.CheckIneffectiveLoop, - "SA4005": c.CheckIneffecitiveFieldAssignments, + "SA4005": c.CheckIneffectiveFieldAssignments, "SA4006": c.CheckUnreadVariableValues, // "SA4007": c.CheckPredeterminedBooleanExprs, "SA4007": nil, @@ -263,6 +264,7 @@ func (c *Checker) Funcs() map[string]lint.Func { "SA4015": c.callChecker(checkMathIntRules), "SA4016": c.CheckSillyBitwiseOps, "SA4017": c.CheckPureFunctions, + "SA4018": c.CheckSelfAssignment, "SA5000": c.CheckNilMaps, "SA5001": c.CheckEarlyDefer, @@ -277,6 +279,7 @@ func (c *Checker) Funcs() map[string]lint.Func { "SA6001": c.CheckMapBytesKey, "SA6002": c.callChecker(checkSyncPoolSizeRules), "SA6003": c.CheckRangeStringRunes, + "SA6004": nil, "SA9000": nil, "SA9001": c.CheckDubiousDeferInChannelRangeLoop, @@ -1137,11 +1140,18 @@ func (c *Checker) CheckEmptyCriticalSection(j *lint.Job) { } } +// cgo produces code like fn(&*_Cvar_kSomeCallbacks) which we don't +// want to flag. +var cgoIdent = regexp.MustCompile(`^_C(func|var)_.+$`) + func (c *Checker) CheckIneffectiveCopy(j *lint.Job) { fn := func(node ast.Node) bool { if unary, ok := node.(*ast.UnaryExpr); ok { - if _, ok := unary.X.(*ast.StarExpr); ok && unary.Op == token.AND { - j.Errorf(unary, "&*x will be simplified to x. It will not copy x.") + if star, ok := unary.X.(*ast.StarExpr); ok && unary.Op == token.AND { + ident, ok := star.X.(*ast.Ident) + if !ok || !cgoIdent.MatchString(ident.Name) { + j.Errorf(unary, "&*x will be simplified to x. It will not copy x.") + } } } @@ -1254,7 +1264,7 @@ func (c *Checker) CheckBenchmarkN(j *lint.Job) { } } -func (c *Checker) CheckIneffecitiveFieldAssignments(j *lint.Job) { +func (c *Checker) CheckIneffectiveFieldAssignments(j *lint.Job) { for _, ssafn := range j.Program.InitialFunctions { // fset := j.Program.SSA.Fset // if fset.File(f.File.Pos()) != fset.File(ssafn.Pos()) { @@ -2558,46 +2568,6 @@ func (c *Checker) checkCalls(j *lint.Job, rules map[string]CallCheck) { } } -func (c *Checker) CheckFlagUsage(j *lint.Job) { - for _, ssafn := range j.Program.InitialFunctions { - for _, block := range ssafn.Blocks { - for _, ins := range block.Instrs { - store, ok := ins.(*ssa.Store) - if !ok { - continue - } - switch addr := store.Addr.(type) { - case *ssa.FieldAddr: - typ := addr.X.Type() - st := deref(typ).Underlying().(*types.Struct) - if types.TypeString(typ, nil) != "*flag.FlagSet" { - continue - } - if st.Field(addr.Field).Name() != "Usage" { - continue - } - case *ssa.Global: - if addr.Pkg.Pkg.Path() != "flag" || addr.Name() != "Usage" { - continue - } - default: - continue - } - fn := unwrapFunction(store.Val) - if fn == nil { - continue - } - for _, oblock := range fn.Blocks { - if hasCallTo(oblock, "os.Exit") { - j.Errorf(store, "the function assigned to Usage shouldn't call os.Exit, but it does") - break - } - } - } - } - } -} - func unwrapFunction(val ssa.Value) *ssa.Function { switch val := val.(type) { case *ssa.Function: @@ -2791,3 +2761,26 @@ func (c *Checker) CheckMapBytesKey(j *lint.Job) { func (c *Checker) CheckRangeStringRunes(j *lint.Job) { sharedcheck.CheckRangeStringRunes(c.nodeFns, j) } + +func (c *Checker) CheckSelfAssignment(j *lint.Job) { + fn := func(node ast.Node) bool { + assign, ok := node.(*ast.AssignStmt) + if !ok { + return true + } + if assign.Tok != token.ASSIGN || len(assign.Lhs) != len(assign.Rhs) { + return true + } + for i, stmt := range assign.Lhs { + rlh := j.Render(stmt) + rrh := j.Render(assign.Rhs[i]) + if rlh == rrh { + j.Errorf(assign, "self-assignment of %s to %s", rrh, rlh) + } + } + return true + } + for _, f := range c.filterGenerated(j.Program.Files) { + ast.Inspect(f, fn) + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/manifest b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/manifest index ad8bcca4e..dad0ba934 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/manifest +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/manifest @@ -5,7 +5,7 @@ "importpath": "github.com/GoASTScanner/gas", "repository": "https://github.com/GoASTScanner/gas", "vcs": "git", - "revision": "1beec25f7754273c9672a3368ea7048d4e73138e", + "revision": "f22c701483ba201fbdb79c3667a28ef6a4e4a25c", "branch": "master", "notests": true }, @@ -17,6 +17,14 @@ "branch": "master", "notests": true }, + { + "importpath": "github.com/alexkohler/nakedret", + "repository": "https://github.com/alexkohler/nakedret", + "vcs": "git", + "revision": "ca8b55b818e14bc0f1f52b714e7474634eadcd34", + "branch": "master", + "notests": true + }, { "importpath": "github.com/client9/misspell", "repository": "https://github.com/client9/misspell", @@ -25,6 +33,14 @@ "branch": "master", "notests": true }, + { + "importpath": "github.com/dnephin/govet", + "repository": "https://github.com/dnephin/govet", + "vcs": "git", + "revision": "4a96d43e39d340b63daa8bc5576985aa599885f6", + "branch": "fork", + "notests": true + }, { "importpath": "github.com/golang/lint", "repository": "https://github.com/golang/lint", @@ -61,7 +77,15 @@ "importpath": "github.com/kisielk/gotool", "repository": "https://github.com/kisielk/gotool", "vcs": "git", - "revision": "0de1eaf82fa3f583ce21fde859f1e7e0c5e9b220", + "revision": "d6ce6262d87e3a4e153e86023ff56ae771554a41", + "branch": "master", + "notests": true + }, + { + "importpath": "github.com/mdempsky/maligned", + "repository": "https://github.com/mdempsky/maligned", + "vcs": "git", + "revision": "08c8e9db1bce03f1af283686c0943fcb75f0109e", "branch": "master", "notests": true }, @@ -81,30 +105,6 @@ "branch": "master", "notests": true }, - { - "importpath": "github.com/mvdan/interfacer", - "repository": "https://github.com/mvdan/interfacer", - "vcs": "git", - "revision": "22c51662ff476dfd97944f74db1b263ed920ee83", - "branch": "master", - "notests": true - }, - { - "importpath": "github.com/mvdan/lint", - "repository": "https://github.com/mvdan/lint", - "vcs": "git", - "revision": "c9cbe299b369cbfea16318baaa037b19a69e45d2", - "branch": "master", - "notests": true - }, - { - "importpath": "github.com/mvdan/unparam", - "repository": "https://github.com/mvdan/unparam", - "vcs": "git", - "revision": "d647bb803b10a6777ee4c6a176416b91fa14713e", - "branch": "master", - "notests": true - }, { "importpath": "github.com/opennota/check", "repository": "https://github.com/opennota/check", @@ -141,7 +141,7 @@ "importpath": "golang.org/x/text/internal/gen", "repository": "https://go.googlesource.com/text", "vcs": "git", - "revision": "cfdf022e86b4ecfb646e1efbd7db175dd623a8fa", + "revision": "bd91bbf73e9a4a801adbfb97133c992678533126", "branch": "master", "path": "internal/gen", "notests": true @@ -150,7 +150,7 @@ "importpath": "golang.org/x/text/internal/triegen", "repository": "https://go.googlesource.com/text", "vcs": "git", - "revision": "cfdf022e86b4ecfb646e1efbd7db175dd623a8fa", + "revision": "bd91bbf73e9a4a801adbfb97133c992678533126", "branch": "master", "path": "internal/triegen", "notests": true @@ -159,7 +159,7 @@ "importpath": "golang.org/x/text/internal/ucd", "repository": "https://go.googlesource.com/text", "vcs": "git", - "revision": "cfdf022e86b4ecfb646e1efbd7db175dd623a8fa", + "revision": "bd91bbf73e9a4a801adbfb97133c992678533126", "branch": "master", "path": "internal/ucd", "notests": true @@ -168,7 +168,7 @@ "importpath": "golang.org/x/text/transform", "repository": "https://go.googlesource.com/text", "vcs": "git", - "revision": "cfdf022e86b4ecfb646e1efbd7db175dd623a8fa", + "revision": "bd91bbf73e9a4a801adbfb97133c992678533126", "branch": "master", "path": "transform", "notests": true @@ -177,7 +177,7 @@ "importpath": "golang.org/x/text/unicode/cldr", "repository": "https://go.googlesource.com/text", "vcs": "git", - "revision": "cfdf022e86b4ecfb646e1efbd7db175dd623a8fa", + "revision": "bd91bbf73e9a4a801adbfb97133c992678533126", "branch": "master", "path": "unicode/cldr", "notests": true @@ -186,7 +186,7 @@ "importpath": "golang.org/x/text/width", "repository": "https://go.googlesource.com/text", "vcs": "git", - "revision": "cfdf022e86b4ecfb646e1efbd7db175dd623a8fa", + "revision": "bd91bbf73e9a4a801adbfb97133c992678533126", "branch": "master", "path": "/width", "notests": true @@ -195,7 +195,7 @@ "importpath": "golang.org/x/tools/cmd/goimports", "repository": "https://go.googlesource.com/tools", "vcs": "git", - "revision": "bce9606b3f617bc6280aab6abbf25962c23f398d", + "revision": "3b1faeda9afbcba128c2d794b38ffe7982141139", "branch": "master", "path": "/cmd/goimports", "notests": true @@ -204,7 +204,7 @@ "importpath": "golang.org/x/tools/cmd/gotype", "repository": "https://go.googlesource.com/tools", "vcs": "git", - "revision": "bce9606b3f617bc6280aab6abbf25962c23f398d", + "revision": "3b1faeda9afbcba128c2d794b38ffe7982141139", "branch": "master", "path": "/cmd/gotype", "notests": true @@ -213,7 +213,7 @@ "importpath": "golang.org/x/tools/container/intsets", "repository": "https://go.googlesource.com/tools", "vcs": "git", - "revision": "bce9606b3f617bc6280aab6abbf25962c23f398d", + "revision": "3b1faeda9afbcba128c2d794b38ffe7982141139", "branch": "master", "path": "/container/intsets", "notests": true @@ -222,7 +222,7 @@ "importpath": "golang.org/x/tools/go/ast/astutil", "repository": "https://go.googlesource.com/tools", "vcs": "git", - "revision": "bce9606b3f617bc6280aab6abbf25962c23f398d", + "revision": "3b1faeda9afbcba128c2d794b38ffe7982141139", "branch": "master", "path": "go/ast/astutil", "notests": true @@ -231,7 +231,7 @@ "importpath": "golang.org/x/tools/go/buildutil", "repository": "https://go.googlesource.com/tools", "vcs": "git", - "revision": "bce9606b3f617bc6280aab6abbf25962c23f398d", + "revision": "3b1faeda9afbcba128c2d794b38ffe7982141139", "branch": "master", "path": "go/buildutil", "notests": true @@ -240,7 +240,7 @@ "importpath": "golang.org/x/tools/go/callgraph", "repository": "https://go.googlesource.com/tools", "vcs": "git", - "revision": "bce9606b3f617bc6280aab6abbf25962c23f398d", + "revision": "3b1faeda9afbcba128c2d794b38ffe7982141139", "branch": "master", "path": "/go/callgraph", "notests": true @@ -249,7 +249,7 @@ "importpath": "golang.org/x/tools/go/gcexportdata", "repository": "https://go.googlesource.com/tools", "vcs": "git", - "revision": "bce9606b3f617bc6280aab6abbf25962c23f398d", + "revision": "3b1faeda9afbcba128c2d794b38ffe7982141139", "branch": "master", "path": "/go/gcexportdata", "notests": true @@ -258,7 +258,7 @@ "importpath": "golang.org/x/tools/go/gcimporter15", "repository": "https://go.googlesource.com/tools", "vcs": "git", - "revision": "bce9606b3f617bc6280aab6abbf25962c23f398d", + "revision": "3b1faeda9afbcba128c2d794b38ffe7982141139", "branch": "master", "path": "/go/gcimporter15", "notests": true @@ -267,7 +267,7 @@ "importpath": "golang.org/x/tools/go/loader", "repository": "https://go.googlesource.com/tools", "vcs": "git", - "revision": "bce9606b3f617bc6280aab6abbf25962c23f398d", + "revision": "3b1faeda9afbcba128c2d794b38ffe7982141139", "branch": "master", "path": "/go/loader", "notests": true @@ -276,7 +276,7 @@ "importpath": "golang.org/x/tools/go/pointer", "repository": "https://go.googlesource.com/tools", "vcs": "git", - "revision": "bce9606b3f617bc6280aab6abbf25962c23f398d", + "revision": "3b1faeda9afbcba128c2d794b38ffe7982141139", "branch": "master", "path": "go/pointer", "notests": true @@ -285,7 +285,7 @@ "importpath": "golang.org/x/tools/go/ssa", "repository": "https://go.googlesource.com/tools", "vcs": "git", - "revision": "bce9606b3f617bc6280aab6abbf25962c23f398d", + "revision": "3b1faeda9afbcba128c2d794b38ffe7982141139", "branch": "master", "path": "/go/ssa", "notests": true @@ -294,7 +294,7 @@ "importpath": "golang.org/x/tools/go/types/typeutil", "repository": "https://go.googlesource.com/tools", "vcs": "git", - "revision": "bce9606b3f617bc6280aab6abbf25962c23f398d", + "revision": "3b1faeda9afbcba128c2d794b38ffe7982141139", "branch": "master", "path": "/go/types/typeutil", "notests": true @@ -303,7 +303,7 @@ "importpath": "golang.org/x/tools/imports", "repository": "https://go.googlesource.com/tools", "vcs": "git", - "revision": "bce9606b3f617bc6280aab6abbf25962c23f398d", + "revision": "3b1faeda9afbcba128c2d794b38ffe7982141139", "branch": "master", "path": "imports", "notests": true @@ -312,7 +312,7 @@ "importpath": "honnef.co/go/tools/callgraph", "repository": "https://github.com/dominikh/go-tools", "vcs": "git", - "revision": "f583b587b6ff1149f9a9b0c16ebdda74da44e1a2", + "revision": "49f44f893d933fd08cd7d67d65ccefa5d7c23329", "branch": "master", "path": "callgraph", "notests": true @@ -321,7 +321,7 @@ "importpath": "honnef.co/go/tools/cmd/gosimple", "repository": "https://github.com/dominikh/go-tools", "vcs": "git", - "revision": "f583b587b6ff1149f9a9b0c16ebdda74da44e1a2", + "revision": "49f44f893d933fd08cd7d67d65ccefa5d7c23329", "branch": "master", "path": "/cmd/gosimple", "notests": true @@ -330,7 +330,7 @@ "importpath": "honnef.co/go/tools/cmd/megacheck", "repository": "https://github.com/dominikh/go-tools", "vcs": "git", - "revision": "f583b587b6ff1149f9a9b0c16ebdda74da44e1a2", + "revision": "49f44f893d933fd08cd7d67d65ccefa5d7c23329", "branch": "master", "path": "/cmd/megacheck", "notests": true @@ -339,7 +339,7 @@ "importpath": "honnef.co/go/tools/cmd/staticcheck", "repository": "https://github.com/dominikh/go-tools", "vcs": "git", - "revision": "f583b587b6ff1149f9a9b0c16ebdda74da44e1a2", + "revision": "49f44f893d933fd08cd7d67d65ccefa5d7c23329", "branch": "master", "path": "/cmd/staticcheck", "notests": true @@ -348,7 +348,7 @@ "importpath": "honnef.co/go/tools/cmd/unused", "repository": "https://github.com/dominikh/go-tools", "vcs": "git", - "revision": "f583b587b6ff1149f9a9b0c16ebdda74da44e1a2", + "revision": "49f44f893d933fd08cd7d67d65ccefa5d7c23329", "branch": "master", "path": "/cmd/unused", "notests": true @@ -357,7 +357,7 @@ "importpath": "honnef.co/go/tools/functions", "repository": "https://github.com/dominikh/go-tools", "vcs": "git", - "revision": "f583b587b6ff1149f9a9b0c16ebdda74da44e1a2", + "revision": "49f44f893d933fd08cd7d67d65ccefa5d7c23329", "branch": "master", "path": "functions", "notests": true @@ -366,7 +366,7 @@ "importpath": "honnef.co/go/tools/gcsizes", "repository": "https://github.com/dominikh/go-tools", "vcs": "git", - "revision": "f583b587b6ff1149f9a9b0c16ebdda74da44e1a2", + "revision": "49f44f893d933fd08cd7d67d65ccefa5d7c23329", "branch": "master", "path": "gcsizes", "notests": true @@ -375,7 +375,7 @@ "importpath": "honnef.co/go/tools/internal/sharedcheck", "repository": "https://github.com/dominikh/go-tools", "vcs": "git", - "revision": "f583b587b6ff1149f9a9b0c16ebdda74da44e1a2", + "revision": "49f44f893d933fd08cd7d67d65ccefa5d7c23329", "branch": "master", "path": "/internal/sharedcheck", "notests": true @@ -384,7 +384,7 @@ "importpath": "honnef.co/go/tools/lint", "repository": "https://github.com/dominikh/go-tools", "vcs": "git", - "revision": "f583b587b6ff1149f9a9b0c16ebdda74da44e1a2", + "revision": "49f44f893d933fd08cd7d67d65ccefa5d7c23329", "branch": "master", "path": "lint", "notests": true @@ -393,7 +393,7 @@ "importpath": "honnef.co/go/tools/simple", "repository": "https://github.com/dominikh/go-tools", "vcs": "git", - "revision": "f583b587b6ff1149f9a9b0c16ebdda74da44e1a2", + "revision": "49f44f893d933fd08cd7d67d65ccefa5d7c23329", "branch": "master", "path": "simple", "notests": true @@ -402,7 +402,7 @@ "importpath": "honnef.co/go/tools/ssa", "repository": "https://github.com/dominikh/go-tools", "vcs": "git", - "revision": "f583b587b6ff1149f9a9b0c16ebdda74da44e1a2", + "revision": "49f44f893d933fd08cd7d67d65ccefa5d7c23329", "branch": "master", "path": "ssa", "notests": true @@ -411,7 +411,7 @@ "importpath": "honnef.co/go/tools/staticcheck", "repository": "https://github.com/dominikh/go-tools", "vcs": "git", - "revision": "f583b587b6ff1149f9a9b0c16ebdda74da44e1a2", + "revision": "49f44f893d933fd08cd7d67d65ccefa5d7c23329", "branch": "master", "path": "staticcheck", "notests": true @@ -420,10 +420,34 @@ "importpath": "honnef.co/go/tools/unused", "repository": "https://github.com/dominikh/go-tools", "vcs": "git", - "revision": "f583b587b6ff1149f9a9b0c16ebdda74da44e1a2", + "revision": "49f44f893d933fd08cd7d67d65ccefa5d7c23329", "branch": "master", "path": "unused", "notests": true + }, + { + "importpath": "mvdan.cc/interfacer", + "repository": "https://github.com/mvdan/interfacer", + "vcs": "git", + "revision": "d7e7372184a059b8fd99d96a593e3811bf989d75", + "branch": "master", + "notests": true + }, + { + "importpath": "mvdan.cc/lint", + "repository": "https://github.com/mvdan/lint", + "vcs": "git", + "revision": "adc824a0674b99099789b6188a058d485eaf61c0", + "branch": "master", + "notests": true + }, + { + "importpath": "mvdan.cc/unparam", + "repository": "https://github.com/mvdan/unparam", + "vcs": "git", + "revision": "6b9a9bf4cdf71fae79104529ee3f16148302cc71", + "branch": "master", + "notests": true } ] } diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/LICENSE b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/LICENSE similarity index 100% rename from vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/LICENSE rename to vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/LICENSE diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/cache.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/check/cache.go similarity index 98% rename from vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/cache.go rename to vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/check/cache.go index ff9d95b9d..757eca55e 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/cache.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/check/cache.go @@ -1,7 +1,7 @@ // Copyright (c) 2015, Daniel Martí // See LICENSE for licensing information -package interfacer +package check import ( "go/ast" diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/check.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/check/check.go similarity index 99% rename from vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/check.go rename to vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/check/check.go index 26bdc9519..f4d3b4037 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/check.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/check/check.go @@ -1,7 +1,7 @@ // Copyright (c) 2015, Daniel Martí // See LICENSE for licensing information -package interfacer +package check // import "mvdan.cc/interfacer/check" import ( "fmt" @@ -16,7 +16,7 @@ import ( "golang.org/x/tools/go/ssa/ssautil" "github.com/kisielk/gotool" - "github.com/mvdan/lint" + "mvdan.cc/lint" ) func toDiscard(usage *varUsage) bool { @@ -111,7 +111,6 @@ type Checker struct { pkgTypes *loader.PackageInfo - fset *token.FileSet funcs []*funcDecl ssaByPos map[token.Pos]*ssa.Function diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/types.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/check/types.go similarity index 99% rename from vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/types.go rename to vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/check/types.go index 6f5145168..393bb0b9f 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/types.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/check/types.go @@ -1,7 +1,7 @@ // Copyright (c) 2015, Daniel Martí // See LICENSE for licensing information -package interfacer +package check import ( "bytes" diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/cmd/interfacer/main.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/main.go similarity index 75% rename from vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/cmd/interfacer/main.go rename to vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/main.go index 7b7fc47a2..bc86977a1 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/interfacer/cmd/interfacer/main.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/interfacer/main.go @@ -1,21 +1,21 @@ // Copyright (c) 2015, Daniel Martí // See LICENSE for licensing information -package main +package main // import "mvdan.cc/interfacer" import ( "flag" "fmt" "os" - "github.com/mvdan/interfacer" + "mvdan.cc/interfacer/check" ) var _ = flag.Bool("v", false, "print the names of packages as they are checked") func main() { flag.Parse() - lines, err := interfacer.CheckArgs(flag.Args()) + lines, err := check.CheckArgs(flag.Args()) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/lint/LICENSE b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/lint/LICENSE similarity index 100% rename from vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/lint/LICENSE rename to vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/lint/LICENSE diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/lint/cmd/metalint/main.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/lint/cmd/metalint/main.go similarity index 92% rename from vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/lint/cmd/metalint/main.go rename to vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/lint/cmd/metalint/main.go index bd0f99a46..ca1646e6f 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/lint/cmd/metalint/main.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/lint/cmd/metalint/main.go @@ -1,7 +1,7 @@ // Copyright (c) 2017, Daniel Martí // See LICENSE for licensing information -package main +package main // import "mvdan.cc/lint/cmd/metalint" import ( "flag" @@ -13,12 +13,12 @@ import ( "golang.org/x/tools/go/ssa" "golang.org/x/tools/go/ssa/ssautil" - "github.com/mvdan/lint" + "mvdan.cc/lint" "github.com/kisielk/gotool" - "github.com/mvdan/interfacer" - unparam "github.com/mvdan/unparam/check" + interfacer "mvdan.cc/interfacer/check" + unparam "mvdan.cc/unparam/check" ) var tests = flag.Bool("tests", false, "include tests") diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/lint/lint.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/lint/lint.go similarity index 93% rename from vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/lint/lint.go rename to vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/lint/lint.go index dea7ae6c0..a16789fad 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/lint/lint.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/lint/lint.go @@ -2,7 +2,7 @@ // See LICENSE for licensing information // Package lint defines common interfaces for Go code checkers. -package lint +package lint // import "mvdan.cc/lint" import ( "go/token" diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/unparam/LICENSE b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/unparam/LICENSE similarity index 100% rename from vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/unparam/LICENSE rename to vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/unparam/LICENSE diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/unparam/check/check.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/unparam/check/check.go similarity index 62% rename from vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/unparam/check/check.go rename to vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/unparam/check/check.go index 65f7d1846..e9c2a99d1 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/github.com/mvdan/unparam/check/check.go +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/unparam/check/check.go @@ -3,7 +3,7 @@ // Package check implements the unparam linter. Note that its API is not // stable. -package check +package check // import "mvdan.cc/unparam/check" import ( "fmt" @@ -12,6 +12,7 @@ import ( "go/parser" "go/token" "go/types" + "io" "os" "path/filepath" "regexp" @@ -25,17 +26,20 @@ import ( "golang.org/x/tools/go/ssa/ssautil" "github.com/kisielk/gotool" - "github.com/mvdan/lint" + "mvdan.cc/lint" ) -func UnusedParams(tests bool, args ...string) ([]string, error) { +func UnusedParams(tests, debug bool, args ...string) ([]string, error) { wd, err := os.Getwd() if err != nil { return nil, err } c := &Checker{ - wd: wd, tests: tests, - cachedDeclCounts: make(map[string]map[string]int), + wd: wd, + tests: tests, + } + if debug { + c.debugLog = os.Stderr } return c.lines(args...) } @@ -46,7 +50,8 @@ type Checker struct { wd string - tests bool + tests bool + debugLog io.Writer cachedDeclCounts map[string]map[string]int } @@ -54,6 +59,8 @@ type Checker struct { var ( _ lint.Checker = (*Checker)(nil) _ lint.WithSSA = (*Checker)(nil) + + skipValue = new(ssa.Value) ) func (c *Checker) lines(args ...string) ([]string, error) { @@ -101,7 +108,14 @@ func (c *Checker) ProgramSSA(prog *ssa.Program) { c.prog = prog } +func (c *Checker) debug(format string, a ...interface{}) { + if c.debugLog != nil { + fmt.Fprintf(c.debugLog, format, a...) + } +} + func (c *Checker) Check() ([]lint.Issue, error) { + c.cachedDeclCounts = make(map[string]map[string]int) wantPkg := make(map[*types.Package]*loader.PackageInfo) for _, info := range c.lprog.InitialPackages() { wantPkg[info.Pkg] = info @@ -121,7 +135,9 @@ funcLoop: if info == nil { // not part of given pkgs continue } + c.debug("func %s\n", fn.String()) if dummyImpl(fn.Blocks[0]) { // panic implementation + c.debug(" skip - dummy implementation\n") continue } for _, edge := range cg.Nodes[fn].In { @@ -130,24 +146,100 @@ funcLoop: default: // called via a parameter or field, type // is set in stone. + c.debug(" skip - type is required via call\n") continue funcLoop } } if c.multipleImpls(info, fn) { + c.debug(" skip - multiple implementations via build tags\n") continue } + + callers := cg.Nodes[fn].In + results := fn.Signature.Results() + // skip exported funcs, as well as those that are + // entirely unused + if !ast.IsExported(fn.Name()) && len(callers) > 0 { + resLoop: + for i := 0; i < results.Len(); i++ { + for _, edge := range callers { + val := edge.Site.Value() + if val == nil { // e.g. go statement + continue + } + for _, instr := range *val.Referrers() { + extract, ok := instr.(*ssa.Extract) + if !ok { + continue resLoop // direct, real use + } + if extract.Index != i { + continue // not the same result param + } + if len(*extract.Referrers()) > 0 { + continue resLoop // real use after extraction + } + } + } + res := results.At(i) + name := paramDesc(i, res) + issues = append(issues, Issue{ + pos: res.Pos(), + msg: fmt.Sprintf("result %s is never used", name), + }) + } + } + + seen := make([]constant.Value, results.Len()) + numRets := 0 + for _, block := range fn.Blocks { + last := block.Instrs[len(block.Instrs)-1] + ret, ok := last.(*ssa.Return) + if !ok { + continue + } + for i, val := range ret.Results { + cnst, ok := val.(*ssa.Const) + switch { + case !ok: + seen[i] = nil + case numRets == 0: + seen[i] = cnst.Value + case seen[i] == nil: + case !constant.Compare(seen[i], token.EQL, cnst.Value): + seen[i] = nil + } + } + numRets++ + } + if numRets > 1 { + for i, val := range seen { + if val == nil { + continue + } + res := results.At(i) + name := paramDesc(i, res) + issues = append(issues, Issue{ + pos: res.Pos(), + msg: fmt.Sprintf("result %s is always %s", name, val.String()), + }) + } + } + for i, par := range fn.Params { if i == 0 && fn.Signature.Recv() != nil { // receiver continue } + c.debug("%s\n", par.String()) switch par.Object().Name() { case "", "_": // unnamed + c.debug(" skip - unnamed\n") continue } reason := "is unused" if cv := receivesSameValue(cg.Nodes[fn].In, par, i); cv != nil { reason = fmt.Sprintf("always receives %v", cv) } else if anyRealUse(par, i) { + c.debug(" skip - used somewhere in the func body\n") continue } issues = append(issues, Issue{ @@ -158,15 +250,25 @@ funcLoop: } // TODO: replace by sort.Slice once we drop Go 1.7 support - sort.Sort(byPos(issues)) + sort.Sort(byNamePos{c.prog.Fset, issues}) return issues, nil } -type byPos []lint.Issue +type byNamePos struct { + fset *token.FileSet + l []lint.Issue +} -func (p byPos) Len() int { return len(p) } -func (p byPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p byPos) Less(i, j int) bool { return p[i].Pos() < p[j].Pos() } +func (p byNamePos) Len() int { return len(p.l) } +func (p byNamePos) Swap(i, j int) { p.l[i], p.l[j] = p.l[j], p.l[i] } +func (p byNamePos) Less(i, j int) bool { + p1 := p.fset.Position(p.l[i].Pos()) + p2 := p.fset.Position(p.l[j].Pos()) + if p1.Filename == p2.Filename { + return p1.Offset < p2.Offset + } + return p1.Filename < p2.Filename +} func receivesSameValue(in []*callgraph.Edge, par *ssa.Parameter, pos int) constant.Value { if ast.IsExported(par.Parent().Name()) { @@ -192,27 +294,47 @@ func receivesSameValue(in []*callgraph.Edge, par *ssa.Parameter, pos int) consta func anyRealUse(par *ssa.Parameter, pos int) bool { refLoop: for _, ref := range *par.Referrers() { - call, ok := ref.(*ssa.Call) - if !ok { + switch x := ref.(type) { + case *ssa.Call: + if x.Call.Value != par.Parent() { + return true // not a recursive call + } + for i, arg := range x.Call.Args { + if arg != par { + continue + } + if i == pos { + // reused directly in a recursive call + continue refLoop + } + } + return true + case *ssa.Store: + if insertedStore(x) { + continue // inserted by go/ssa, not from the code + } + return true + default: return true } - if call.Call.Value != par.Parent() { - return true // not a recursive call - } - for i, arg := range call.Call.Args { - if arg != par { - continue - } - if i == pos { - // reused directly in a recursive call - continue refLoop - } - } - return true } return false } +func insertedStore(instr ssa.Instruction) bool { + if instr.Pos() != token.NoPos { + return false + } + store, ok := instr.(*ssa.Store) + if !ok { + return false + } + alloc, ok := store.Addr.(*ssa.Alloc) + // we want exactly one use of this alloc value for it to be + // inserted by ssa and dummy - the alloc instruction itself. + return ok && len(*alloc.Referrers()) == 1 +} + var rxHarmlessCall = regexp.MustCompile(`(?i)\b(log(ger)?|errors)\b|\bf?print`) // dummyImpl reports whether a block is a dummy implementation. This is @@ -221,11 +343,15 @@ var rxHarmlessCall = regexp.MustCompile(`(?i)\b(log(ger)?|errors)\b|\bf?print`) func dummyImpl(blk *ssa.BasicBlock) bool { var ops [8]*ssa.Value for _, instr := range blk.Instrs { + if insertedStore(instr) { + continue // inserted by go/ssa, not from the code + } for _, val := range instr.Operands(ops[:0]) { switch x := (*val).(type) { case nil, *ssa.Const, *ssa.ChangeType, *ssa.Alloc, *ssa.MakeInterface, *ssa.Function, - *ssa.Global, *ssa.IndexAddr, *ssa.Slice: + *ssa.Global, *ssa.IndexAddr, *ssa.Slice, + *ssa.UnOp: case *ssa.Call: if rxHarmlessCall.MatchString(x.Call.Value.String()) { continue @@ -322,3 +448,11 @@ func (c *Checker) multipleImpls(info *loader.PackageInfo, fn *ssa.Function) bool } return count[name] > 1 } + +func paramDesc(i int, v *types.Var) string { + name := v.Name() + if name != "" { + return name + } + return fmt.Sprintf("%d (%s)", i, v.Type().String()) +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/unparam/main.go b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/unparam/main.go new file mode 100644 index 000000000..524736df7 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/_linters/src/mvdan.cc/unparam/main.go @@ -0,0 +1,33 @@ +// Copyright (c) 2017, Daniel Martí +// See LICENSE for licensing information + +package main // import "mvdan.cc/unparam" + +import ( + "flag" + "fmt" + "os" + + "mvdan.cc/unparam/check" +) + +var ( + tests = flag.Bool("tests", true, "include tests") + debug = flag.Bool("debug", false, "debug prints") +) + +func main() { + flag.Usage = func() { + fmt.Fprintln(os.Stderr, "usage: unparam [flags] [package ...]") + flag.PrintDefaults() + } + flag.Parse() + warns, err := check.UnusedParams(*tests, *debug, flag.Args()...) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + for _, warn := range warns { + fmt.Println(warn) + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/aggregate.go b/vendor/src/github.com/alecthomas/gometalinter/aggregate.go index 18f8f38e5..f4b44e4b0 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/aggregate.go +++ b/vendor/src/github.com/alecthomas/gometalinter/aggregate.go @@ -5,27 +5,21 @@ import ( "strings" ) -type ( - issueKey struct { - path string - line, col int - message string - } - - multiIssue struct { - *Issue - linterNames []string - } -) - -func maybeAggregateIssues(issues chan *Issue) chan *Issue { - if !config.Aggregate { - return issues - } - return aggregateIssues(issues) +type issueKey struct { + path string + line, col int + message string } -func aggregateIssues(issues chan *Issue) chan *Issue { +type multiIssue struct { + *Issue + linterNames []string +} + +// AggregateIssueChan reads issues from a channel, aggregates issues which have +// the same file, line, vol, and message, and returns aggregated issues on +// a new channel. +func AggregateIssueChan(issues chan *Issue) chan *Issue { out := make(chan *Issue, 1000000) issueMap := make(map[issueKey]*multiIssue) go func() { diff --git a/vendor/src/github.com/alecthomas/gometalinter/checkstyle.go b/vendor/src/github.com/alecthomas/gometalinter/checkstyle.go index b5f09dc3b..5122604e7 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/checkstyle.go +++ b/vendor/src/github.com/alecthomas/gometalinter/checkstyle.go @@ -4,7 +4,7 @@ import ( "encoding/xml" "fmt" - "gopkg.in/alecthomas/kingpin.v3-unstable" + kingpin "gopkg.in/alecthomas/kingpin.v3-unstable" ) type checkstyleOutput struct { diff --git a/vendor/src/github.com/alecthomas/gometalinter/config.go b/vendor/src/github.com/alecthomas/gometalinter/config.go index 9fb7f05c9..bdcb4d04f 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/config.go +++ b/vendor/src/github.com/alecthomas/gometalinter/config.go @@ -8,7 +8,7 @@ import ( ) // Config for gometalinter. This can be loaded from a JSON file with --config. -type Config struct { // nolint: aligncheck +type Config struct { // nolint: maligned // A map from linter name -> . // // For backwards compatibility, the value stored in the JSON blob can also @@ -51,6 +51,11 @@ type Config struct { // nolint: aligncheck EnableGC bool Aggregate bool EnableAll bool + + // Warn if a nolint directive was never matched to a linter issue + WarnUnmatchedDirective bool + + formatTemplate *template.Template } type StringOrLinterConfig LinterConfig @@ -58,7 +63,8 @@ type StringOrLinterConfig LinterConfig func (c *StringOrLinterConfig) UnmarshalJSON(raw []byte) error { var linterConfig LinterConfig // first try to un-marshall directly into struct - if err := json.Unmarshal(raw, &linterConfig); err == nil { + origErr := json.Unmarshal(raw, &linterConfig) + if origErr == nil { *c = StringOrLinterConfig(linterConfig) return nil } @@ -66,7 +72,7 @@ func (c *StringOrLinterConfig) UnmarshalJSON(raw []byte) error { // i.e. bytes didn't represent the struct, treat them as a string var linterSpec string if err := json.Unmarshal(raw, &linterSpec); err != nil { - return err + return origErr } linter, err := parseLinterConfigSpec("", linterSpec) if err != nil { @@ -93,18 +99,16 @@ func (td *jsonDuration) Duration() time.Duration { return time.Duration(*td) } -// TODO: should be a field on Config struct -var formatTemplate = &template.Template{} - var sortKeys = []string{"none", "path", "line", "column", "severity", "message", "linter"} // Configuration defaults. var config = &Config{ - Format: "{{.Path}}:{{.Line}}:{{if .Col}}{{.Col}}{{end}}:{{.Severity}}: {{.Message}} ({{.Linter}})", + Format: DefaultIssueFormat, Linters: map[string]StringOrLinterConfig{}, Severity: map[string]string{ "gotype": "error", + "gotypex": "error", "test": "error", "testify": "error", "vet": "error", diff --git a/vendor/src/github.com/alecthomas/gometalinter/config_test.go b/vendor/src/github.com/alecthomas/gometalinter/config_test.go new file mode 100644 index 000000000..f7a4e9378 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/config_test.go @@ -0,0 +1,21 @@ +package main + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLinterConfigUnmarshalJSON(t *testing.T) { + source := `{ + "Command": "/bin/custom", + "PartitionStrategy": "directories" + }` + var config StringOrLinterConfig + err := json.Unmarshal([]byte(source), &config) + require.NoError(t, err) + assert.Equal(t, "/bin/custom", config.Command) + assert.Equal(t, functionName(partitionPathsAsDirectories), functionName(config.PartitionStrategy)) +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/directives.go b/vendor/src/github.com/alecthomas/gometalinter/directives.go index bd3ffa31a..10f2903d2 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/directives.go +++ b/vendor/src/github.com/alecthomas/gometalinter/directives.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "go/ast" "go/parser" "go/token" @@ -14,6 +15,7 @@ type ignoredRange struct { col int start, end int linters []string + matched bool } func (i *ignoredRange) matches(issue *Issue) bool { @@ -35,6 +37,14 @@ func (i *ignoredRange) near(col, start int) bool { return col == i.col && i.end == start-1 } +func (i *ignoredRange) String() string { + linters := strings.Join(i.linters, ",") + if len(i.linters) == 0 { + linters = "all" + } + return fmt.Sprintf("%s:%d-%d", linters, i.start, i.end) +} + type ignoredRanges []*ignoredRange func (ir ignoredRanges) Len() int { return len(ir) } @@ -66,12 +76,43 @@ func (d *directiveParser) IsIgnored(issue *Issue) bool { d.lock.Unlock() for _, r := range ranges { if r.matches(issue) { + debug("nolint: matched %s to issue %s", r, issue) + r.matched = true return true } } return false } +// Unmatched returns all the ranges which were never used to ignore an issue +func (d *directiveParser) Unmatched() map[string]ignoredRanges { + unmatched := map[string]ignoredRanges{} + for path, ranges := range d.files { + for _, ignore := range ranges { + if !ignore.matched { + unmatched[path] = append(unmatched[path], ignore) + } + } + } + return unmatched +} + +// LoadFiles from a list of directories +func (d *directiveParser) LoadFiles(paths []string) error { + d.lock.Lock() + defer d.lock.Unlock() + filenames, err := pathsToFileGlobs(paths) + if err != nil { + return err + } + for _, filename := range filenames { + ranges := d.parseFile(filename) + sort.Sort(ranges) + d.files[filename] = ranges + } + return nil +} + // Takes a set of ignoredRanges, determines if they immediately precede a statement // construct, and expands the range to include that construct. Why? So you can // precede a function or struct with //nolint @@ -150,7 +191,28 @@ func filterIssuesViaDirectives(directives *directiveParser, issues chan *Issue) out <- issue } } + + if config.WarnUnmatchedDirective { + for _, issue := range warnOnUnusedDirective(directives) { + out <- issue + } + } close(out) }() return out } + +func warnOnUnusedDirective(directives *directiveParser) []*Issue { + out := []*Issue{} + for path, ranges := range directives.Unmatched() { + for _, ignore := range ranges { + issue, _ := NewIssue("nolint", config.formatTemplate) + issue.Path = path + issue.Line = ignore.start + issue.Col = ignore.col + issue.Message = "nolint directive did not match any issue" + out = append(out, issue) + } + } + return out +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/directives_test.go b/vendor/src/github.com/alecthomas/gometalinter/directives_test.go index 06ab7d0f9..f0e3b267b 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/directives_test.go +++ b/vendor/src/github.com/alecthomas/gometalinter/directives_test.go @@ -1 +1,42 @@ package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIgnoreRangeMatch(t *testing.T) { + var testcases = []struct { + doc string + issue Issue + linters []string + expected bool + }{ + { + doc: "unmatched line", + issue: Issue{Line: 100}, + }, + { + doc: "matched line, all linters", + issue: Issue{Line: 5}, + expected: true, + }, + { + doc: "matched line, unmatched linter", + issue: Issue{Line: 5}, + linters: []string{"vet"}, + }, + { + doc: "matched line and linters", + issue: Issue{Line: 20, Linter: "vet"}, + linters: []string{"vet"}, + expected: true, + }, + } + + for _, testcase := range testcases { + ir := ignoredRange{col: 20, start: 5, end: 20, linters: testcase.linters} + assert.Equal(t, testcase.expected, ir.matches(&testcase.issue), testcase.doc) + } +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/execute.go b/vendor/src/github.com/alecthomas/gometalinter/execute.go index 8a45e5889..57267a5d6 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/execute.go +++ b/vendor/src/github.com/alecthomas/gometalinter/execute.go @@ -8,14 +8,13 @@ import ( "path/filepath" "reflect" "regexp" - "sort" "strconv" "strings" "sync" "time" "github.com/google/shlex" - "gopkg.in/alecthomas/kingpin.v3-unstable" + kingpin "gopkg.in/alecthomas/kingpin.v3-unstable" ) type Vars map[string]string @@ -41,35 +40,8 @@ func (v Vars) Replace(s string) string { return s } -// Severity of linter message. -type Severity string - -// Linter message severity levels. -const ( // nolint: deadcode - Error Severity = "error" - Warning Severity = "warning" -) - -type Issue struct { - Linter string `json:"linter"` - Severity Severity `json:"severity"` - Path string `json:"path"` - Line int `json:"line"` - Col int `json:"col"` - Message string `json:"message"` -} - -func (i *Issue) String() string { - buf := new(bytes.Buffer) - err := formatTemplate.Execute(buf, i) - kingpin.FatalIfError(err, "Invalid output format") - return buf.String() -} - type linterState struct { *Linter - id int - paths []string issues chan *Issue vars Vars exclude *regexp.Regexp @@ -77,26 +49,34 @@ type linterState struct { deadline <-chan time.Time } -func (l *linterState) Partitions() ([][]string, error) { - command := l.vars.Replace(l.Command) - cmdArgs, err := parseCommand(command) +func (l *linterState) Partitions(paths []string) ([][]string, error) { + cmdArgs, err := parseCommand(l.command()) if err != nil { return nil, err } - parts, err := l.Linter.PartitionStrategy(cmdArgs, l.paths) + parts, err := l.Linter.PartitionStrategy(cmdArgs, paths) if err != nil { return nil, err } return parts, nil } +func (l *linterState) command() string { + return l.vars.Replace(l.Command) +} + func runLinters(linters map[string]*Linter, paths []string, concurrency int, exclude, include *regexp.Regexp) (chan *Issue, chan error) { errch := make(chan error, len(linters)) concurrencych := make(chan bool, concurrency) incomingIssues := make(chan *Issue, 1000000) - processedIssues := filterIssuesViaDirectives( - newDirectiveParser(), - maybeSortIssues(maybeAggregateIssues(incomingIssues))) + + directiveParser := newDirectiveParser() + if config.WarnUnmatchedDirective { + directiveParser.LoadFiles(paths) + } + + processedIssues := maybeSortIssues(filterIssuesViaDirectives( + directiveParser, maybeAggregateIssues(incomingIssues))) vars := Vars{ "duplthreshold": fmt.Sprintf("%d", config.DuplThreshold), @@ -106,9 +86,11 @@ func runLinters(linters map[string]*Linter, paths []string, concurrency int, exc "min_occurrences": fmt.Sprintf("%d", config.MinOccurrences), "min_const_length": fmt.Sprintf("%d", config.MinConstLength), "tests": "", + "not_tests": "true", } if config.Test { - vars["tests"] = "-t" + vars["tests"] = "true" + vars["not_tests"] = "" } wg := &sync.WaitGroup{} @@ -118,25 +100,24 @@ func runLinters(linters map[string]*Linter, paths []string, concurrency int, exc state := &linterState{ Linter: linter, issues: incomingIssues, - paths: paths, vars: vars, exclude: exclude, include: include, deadline: deadline, } - partitions, err := state.Partitions() + partitions, err := state.Partitions(paths) if err != nil { errch <- err continue } for _, args := range partitions { wg.Add(1) + concurrencych <- true // Call the goroutine with a copy of the args array so that the // contents of the array are not modified by the next iteration of // the above for loop go func(id int, args []string) { - concurrencych <- true err := executeLinter(id, state, args) if err != nil { errch <- err @@ -243,7 +224,9 @@ func processOutput(dbg debugFunction, state *linterState, out []byte) { group = append(group, fragment) } - issue := &Issue{Line: 1, Linter: state.Linter.Name} + issue, err := NewIssue(state.Linter.Name, config.formatTemplate) + kingpin.FatalIfError(err, "Invalid output format") + for i, name := range re.SubexpNames() { if group[i] == nil { continue @@ -279,8 +262,6 @@ func processOutput(dbg debugFunction, state *linterState, out []byte) { } if sev, ok := config.Severity[state.Name]; ok { issue.Severity = Severity(sev) - } else { - issue.Severity = Warning } if state.exclude != nil && state.exclude.MatchString(issue.String()) { continue @@ -323,66 +304,16 @@ func resolvePath(path string) string { return path } -type sortedIssues struct { - issues []*Issue - order []string -} - -func (s *sortedIssues) Len() int { return len(s.issues) } -func (s *sortedIssues) Swap(i, j int) { s.issues[i], s.issues[j] = s.issues[j], s.issues[i] } - -// nolint: gocyclo -func (s *sortedIssues) Less(i, j int) bool { - l, r := s.issues[i], s.issues[j] - for _, key := range s.order { - switch key { - case "path": - if l.Path > r.Path { - return false - } - case "line": - if l.Line > r.Line { - return false - } - case "column": - if l.Col > r.Col { - return false - } - case "severity": - if l.Severity > r.Severity { - return false - } - case "message": - if l.Message > r.Message { - return false - } - case "linter": - if l.Linter > r.Linter { - return false - } - } - } - return true -} - func maybeSortIssues(issues chan *Issue) chan *Issue { if reflect.DeepEqual([]string{"none"}, config.Sort) { return issues } - out := make(chan *Issue, 1000000) - sorted := &sortedIssues{ - issues: []*Issue{}, - order: config.Sort, - } - go func() { - for issue := range issues { - sorted.issues = append(sorted.issues, issue) - } - sort.Sort(sorted) - for _, issue := range sorted.issues { - out <- issue - } - close(out) - }() - return out + return SortIssueChan(issues, config.Sort) +} + +func maybeAggregateIssues(issues chan *Issue) chan *Issue { + if !config.Aggregate { + return issues + } + return AggregateIssueChan(issues) } diff --git a/vendor/src/github.com/alecthomas/gometalinter/execute_test.go b/vendor/src/github.com/alecthomas/gometalinter/execute_test.go index 15fb0693a..cc2703ac9 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/execute_test.go +++ b/vendor/src/github.com/alecthomas/gometalinter/execute_test.go @@ -1,29 +1,67 @@ package main import ( - "sort" "testing" - "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" ) -func TestSortedIssues(t *testing.T) { - actual := []*Issue{ - {Path: "b.go", Line: 5}, - {Path: "a.go", Line: 3}, - {Path: "b.go", Line: 1}, - {Path: "a.go", Line: 1}, +func TestLinterStateCommand(t *testing.T) { + varsDefault := Vars{"tests": "", "not_tests": "true"} + varsWithTest := Vars{"tests": "true", "not_tests": ""} + + var testcases = []struct { + linter string + vars Vars + expected string + }{ + { + linter: "errcheck", + vars: varsWithTest, + expected: `errcheck -abspath `, + }, + { + linter: "errcheck", + vars: varsDefault, + expected: `errcheck -abspath -ignoretests`, + }, + { + linter: "gotype", + vars: varsDefault, + expected: `gotype -e `, + }, + { + linter: "gotype", + vars: varsWithTest, + expected: `gotype -e -t`, + }, + { + linter: "structcheck", + vars: varsDefault, + expected: `structcheck `, + }, + { + linter: "structcheck", + vars: varsWithTest, + expected: `structcheck -t`, + }, + { + linter: "unparam", + vars: varsDefault, + expected: `unparam -tests=false`, + }, + { + linter: "unparam", + vars: varsWithTest, + expected: `unparam `, + }, } - issues := &sortedIssues{ - issues: actual, - order: []string{"path", "line"}, + + for _, testcase := range testcases { + ls := linterState{ + Linter: getLinterByName(testcase.linter, LinterConfig{}), + vars: testcase.vars, + } + assert.Equal(t, testcase.expected, ls.command()) } - sort.Sort(issues) - expected := []*Issue{ - {Path: "a.go", Line: 1}, - {Path: "a.go", Line: 3}, - {Path: "b.go", Line: 1}, - {Path: "b.go", Line: 5}, - } - require.Equal(t, expected, actual) } diff --git a/vendor/src/github.com/alecthomas/gometalinter/issue.go b/vendor/src/github.com/alecthomas/gometalinter/issue.go new file mode 100644 index 000000000..982f45047 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/issue.go @@ -0,0 +1,114 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "sort" + "strings" + "text/template" +) + +// DefaultIssueFormat used to print an issue +const DefaultIssueFormat = "{{.Path}}:{{.Line}}:{{if .Col}}{{.Col}}{{end}}:{{.Severity}}: {{.Message}} ({{.Linter}})" + +// Severity of linter message +type Severity string + +// Linter message severity levels. +const ( + Error Severity = "error" + Warning Severity = "warning" +) + +type Issue struct { + Linter string `json:"linter"` + Severity Severity `json:"severity"` + Path string `json:"path"` + Line int `json:"line"` + Col int `json:"col"` + Message string `json:"message"` + formatTmpl *template.Template +} + +// NewIssue returns a new issue. Returns an error if formatTmpl is not a valid +// template for an Issue. +func NewIssue(linter string, formatTmpl *template.Template) (*Issue, error) { + issue := &Issue{ + Line: 1, + Severity: Warning, + Linter: linter, + formatTmpl: formatTmpl, + } + err := formatTmpl.Execute(ioutil.Discard, issue) + return issue, err +} + +func (i *Issue) String() string { + if i.formatTmpl == nil { + col := "" + if i.Col != 0 { + col = fmt.Sprintf("%d", i.Col) + } + return fmt.Sprintf("%s:%d:%s:%s: %s (%s)", strings.TrimSpace(i.Path), i.Line, col, i.Severity, strings.TrimSpace(i.Message), i.Linter) + } + buf := new(bytes.Buffer) + _ = i.formatTmpl.Execute(buf, i) + return buf.String() +} + +type sortedIssues struct { + issues []*Issue + order []string +} + +func (s *sortedIssues) Len() int { return len(s.issues) } +func (s *sortedIssues) Swap(i, j int) { s.issues[i], s.issues[j] = s.issues[j], s.issues[i] } + +func (s *sortedIssues) Less(i, j int) bool { + l, r := s.issues[i], s.issues[j] + return CompareIssue(*l, *r, s.order) +} + +// CompareIssue two Issues and return true if left should sort before right +// nolint: gocyclo +func CompareIssue(l, r Issue, order []string) bool { + for _, key := range order { + switch { + case key == "path" && l.Path != r.Path: + return l.Path < r.Path + case key == "line" && l.Line != r.Line: + return l.Line < r.Line + case key == "column" && l.Col != r.Col: + return l.Col < r.Col + case key == "severity" && l.Severity != r.Severity: + return l.Severity < r.Severity + case key == "message" && l.Message != r.Message: + return l.Message < r.Message + case key == "linter" && l.Linter != r.Linter: + return l.Linter < r.Linter + } + } + return true +} + +// SortIssueChan reads issues from one channel, sorts them, and returns them to another +// channel +func SortIssueChan(issues chan *Issue, order []string) chan *Issue { + out := make(chan *Issue, 1000000) + sorted := &sortedIssues{ + issues: []*Issue{}, + order: order, + } + go func() { + for issue := range issues { + sorted.issues = append(sorted.issues, issue) + } + sort.Sort(sorted) + for _, issue := range sorted.issues { + out <- issue + } + close(out) + }() + return out +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/issue_test.go b/vendor/src/github.com/alecthomas/gometalinter/issue_test.go new file mode 100644 index 000000000..2d456d88d --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/issue_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSortedIssues(t *testing.T) { + actual := []*Issue{ + {Path: "b.go", Line: 5, Col: 1}, + {Path: "a.go", Line: 3, Col: 2}, + {Path: "b.go", Line: 1, Col: 3}, + {Path: "a.go", Line: 1, Col: 4}, + } + issues := &sortedIssues{ + issues: actual, + order: []string{"path", "line", "column"}, + } + sort.Sort(issues) + expected := []*Issue{ + {Path: "a.go", Line: 1, Col: 4}, + {Path: "a.go", Line: 3, Col: 2}, + {Path: "b.go", Line: 1, Col: 3}, + {Path: "b.go", Line: 5, Col: 1}, + } + require.Equal(t, expected, actual) +} + +func TestCompareOrderWithMessage(t *testing.T) { + order := []string{"path", "line", "column", "message"} + issueM := Issue{Path: "file.go", Message: "message"} + issueU := Issue{Path: "file.go", Message: "unknown"} + + assert.True(t, CompareIssue(issueM, issueU, order)) + assert.False(t, CompareIssue(issueU, issueM, order)) +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/linters.go b/vendor/src/github.com/alecthomas/gometalinter/linters.go index 0b4e81a28..c1f45a74a 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/linters.go +++ b/vendor/src/github.com/alecthomas/gometalinter/linters.go @@ -8,11 +8,10 @@ import ( "sort" "strings" - "gopkg.in/alecthomas/kingpin.v3-unstable" + kingpin "gopkg.in/alecthomas/kingpin.v3-unstable" ) type LinterConfig struct { - Name string Command string Pattern string InstallFrom string @@ -23,11 +22,12 @@ type LinterConfig struct { type Linter struct { LinterConfig + Name string regex *regexp.Regexp } // NewLinter returns a new linter from a config -func NewLinter(config LinterConfig) (*Linter, error) { +func NewLinter(name string, config LinterConfig) (*Linter, error) { if p, ok := predefinedPatterns[config.Pattern]; ok { config.Pattern = p } @@ -36,10 +36,11 @@ func NewLinter(config LinterConfig) (*Linter, error) { return nil, err } if config.PartitionStrategy == nil { - config.PartitionStrategy = partitionToMaxArgSize + config.PartitionStrategy = partitionPathsAsDirectories } return &Linter{ LinterConfig: config, + Name: name, regex: regex, }, nil } @@ -61,7 +62,17 @@ func getLinterByName(name string, overrideConf LinterConfig) *Linter { if val := overrideConf.Pattern; val != "" { conf.Pattern = val } - linter, _ := NewLinter(conf) + if val := overrideConf.InstallFrom; val != "" { + conf.InstallFrom = val + } + if overrideConf.IsFast { + conf.IsFast = true + } + if val := overrideConf.PartitionStrategy; val != nil { + conf.PartitionStrategy = val + } + + linter, _ := NewLinter(name, conf) return linter } @@ -73,7 +84,9 @@ func parseLinterConfigSpec(name string, spec string) (LinterConfig, error) { config := defaultLinters[name] config.Command, config.Pattern = parts[0], parts[1] - config.Name = name + if predefined, ok := predefinedPatterns[config.Pattern]; ok { + config.Pattern = predefined + } return config, nil } @@ -154,9 +167,9 @@ func installLinters() { func getDefaultLinters() []*Linter { out := []*Linter{} - for _, config := range defaultLinters { - linter, err := NewLinter(config) - kingpin.FatalIfError(err, "invalid linter %q", config.Name) + for name, config := range defaultLinters { + linter, err := NewLinter(name, config) + kingpin.FatalIfError(err, "invalid linter %q", name) out = append(out, linter) } return out @@ -172,226 +185,228 @@ func defaultEnabled() []string { return enabled } +func validateLinters(linters map[string]*Linter, config *Config) error { + var unknownLinters []string + for name := range linters { + if _, isDefault := defaultLinters[name]; !isDefault { + if _, isCustom := config.Linters[name]; !isCustom { + unknownLinters = append(unknownLinters, name) + } + } + } + if len(unknownLinters) > 0 { + return fmt.Errorf("unknown linters: %s", strings.Join(unknownLinters, ", ")) + } + return nil +} + const vetPattern = `^(?:vet:.*?\.go:\s+(?P.*?\.go):(?P\d+):(?P\d+):\s*(?P.*))|(?:(?P.*?\.go):(?P\d+):\s*(?P.*))$` var defaultLinters = map[string]LinterConfig{ - "aligncheck": { - Name: "aligncheck", - Command: "aligncheck", + "maligned": { + Command: "maligned", Pattern: `^(?:[^:]+: )?(?P.*?\.go):(?P\d+):(?P\d+):\s*(?P.+)$`, - InstallFrom: "github.com/opennota/check/cmd/aligncheck", - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + InstallFrom: "github.com/mdempsky/maligned", + PartitionStrategy: partitionPathsAsPackages, defaultEnabled: true, }, "deadcode": { - Name: "deadcode", Command: "deadcode", Pattern: `^deadcode: (?P.*?\.go):(?P\d+):(?P\d+):\s*(?P.*)$`, InstallFrom: "github.com/tsenart/deadcode", - PartitionStrategy: partitionToMaxArgSize, + PartitionStrategy: partitionPathsAsDirectories, defaultEnabled: true, }, "dupl": { - Name: "dupl", Command: `dupl -plumbing -threshold {duplthreshold}`, Pattern: `^(?P.*?\.go):(?P\d+)-\d+:\s*(?P.*)$`, InstallFrom: "github.com/mibk/dupl", - PartitionStrategy: partitionToMaxArgSizeWithFileGlobs, + PartitionStrategy: partitionPathsAsFiles, IsFast: true, }, "errcheck": { - Name: "errcheck", - Command: `errcheck -abspath`, + Command: `errcheck -abspath {not_tests=-ignoretests}`, Pattern: `PATH:LINE:COL:MESSAGE`, InstallFrom: "github.com/kisielk/errcheck", - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + PartitionStrategy: partitionPathsAsPackages, defaultEnabled: true, }, "gas": { - Name: "gas", Command: `gas -fmt=csv`, Pattern: `^(?P.*?\.go),(?P\d+),(?P[^,]+,[^,]+,[^,]+)`, InstallFrom: "github.com/GoASTScanner/gas", - PartitionStrategy: partitionToMaxArgSize, + PartitionStrategy: partitionPathsAsDirectories, defaultEnabled: true, IsFast: true, }, "goconst": { - Name: "goconst", Command: `goconst -min-occurrences {min_occurrences} -min-length {min_const_length}`, Pattern: `PATH:LINE:COL:MESSAGE`, InstallFrom: "github.com/jgautheron/goconst/cmd/goconst", - PartitionStrategy: partitionToMaxArgSize, + PartitionStrategy: partitionPathsAsDirectories, defaultEnabled: true, IsFast: true, }, "gocyclo": { - Name: "gocyclo", Command: `gocyclo -over {mincyclo}`, Pattern: `^(?P\d+)\s+\S+\s(?P\S+)\s+(?P.*?\.go):(?P\d+):(\d+)$`, InstallFrom: "github.com/alecthomas/gocyclo", - PartitionStrategy: partitionToMaxArgSize, + PartitionStrategy: partitionPathsAsDirectories, defaultEnabled: true, IsFast: true, }, "gofmt": { - Name: "gofmt", Command: `gofmt -l -s`, Pattern: `^(?P.*?\.go)$`, - PartitionStrategy: partitionToMaxArgSizeWithFileGlobs, + PartitionStrategy: partitionPathsAsFiles, IsFast: true, }, "goimports": { - Name: "goimports", Command: `goimports -l`, Pattern: `^(?P.*?\.go)$`, InstallFrom: "golang.org/x/tools/cmd/goimports", - PartitionStrategy: partitionToMaxArgSizeWithFileGlobs, + PartitionStrategy: partitionPathsAsFiles, IsFast: true, }, "golint": { - Name: "golint", Command: `golint -min_confidence {min_confidence}`, Pattern: `PATH:LINE:COL:MESSAGE`, InstallFrom: "github.com/golang/lint/golint", - PartitionStrategy: partitionToMaxArgSize, + PartitionStrategy: partitionPathsAsDirectories, defaultEnabled: true, IsFast: true, }, "gosimple": { - Name: "gosimple", Command: `gosimple`, Pattern: `PATH:LINE:COL:MESSAGE`, InstallFrom: "honnef.co/go/tools/cmd/gosimple", - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + PartitionStrategy: partitionPathsAsPackages, }, "gotype": { - Name: "gotype", Command: `gotype -e {tests=-t}`, Pattern: `PATH:LINE:COL:MESSAGE`, InstallFrom: "golang.org/x/tools/cmd/gotype", - PartitionStrategy: partitionToMaxArgSize, + PartitionStrategy: partitionPathsByDirectory, + defaultEnabled: true, + IsFast: true, + }, + "gotypex": { + Command: `gotype -e -x`, + Pattern: `PATH:LINE:COL:MESSAGE`, + InstallFrom: "golang.org/x/tools/cmd/gotype", + PartitionStrategy: partitionPathsByDirectory, defaultEnabled: true, IsFast: true, }, "ineffassign": { - Name: "ineffassign", Command: `ineffassign -n`, Pattern: `PATH:LINE:COL:MESSAGE`, InstallFrom: "github.com/gordonklaus/ineffassign", - PartitionStrategy: partitionToMaxArgSize, + PartitionStrategy: partitionPathsAsDirectories, defaultEnabled: true, IsFast: true, }, "interfacer": { - Name: "interfacer", Command: `interfacer`, Pattern: `PATH:LINE:COL:MESSAGE`, - InstallFrom: "github.com/mvdan/interfacer/cmd/interfacer", - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + InstallFrom: "mvdan.cc/interfacer", + PartitionStrategy: partitionPathsAsPackages, defaultEnabled: true, }, "lll": { - Name: "lll", Command: `lll -g -l {maxlinelength}`, Pattern: `PATH:LINE:MESSAGE`, InstallFrom: "github.com/walle/lll/cmd/lll", - PartitionStrategy: partitionToMaxArgSizeWithFileGlobs, + PartitionStrategy: partitionPathsAsFiles, IsFast: true, }, "megacheck": { - Name: "megacheck", Command: `megacheck`, Pattern: `PATH:LINE:COL:MESSAGE`, InstallFrom: "honnef.co/go/tools/cmd/megacheck", - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + PartitionStrategy: partitionPathsAsPackages, defaultEnabled: true, }, "misspell": { - Name: "misspell", Command: `misspell -j 1`, Pattern: `PATH:LINE:COL:MESSAGE`, InstallFrom: "github.com/client9/misspell/cmd/misspell", - PartitionStrategy: partitionToMaxArgSizeWithFileGlobs, + PartitionStrategy: partitionPathsAsFiles, IsFast: true, }, + "nakedret": { + Command: `nakedret`, + Pattern: `^(?P.*?\.go):(?P\d+)\s*(?P.*)$`, + InstallFrom: "github.com/alexkohler/nakedret", + PartitionStrategy: partitionPathsAsDirectories, + }, "safesql": { - Name: "safesql", Command: `safesql`, Pattern: `^- (?P.*?\.go):(?P\d+):(?P\d+)$`, InstallFrom: "github.com/stripe/safesql", - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + PartitionStrategy: partitionPathsAsPackages, }, "staticcheck": { - Name: "staticcheck", Command: `staticcheck`, Pattern: `PATH:LINE:COL:MESSAGE`, InstallFrom: "honnef.co/go/tools/cmd/staticcheck", - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + PartitionStrategy: partitionPathsAsPackages, }, "structcheck": { - Name: "structcheck", Command: `structcheck {tests=-t}`, Pattern: `^(?:[^:]+: )?(?P.*?\.go):(?P\d+):(?P\d+):\s*(?P.+)$`, InstallFrom: "github.com/opennota/check/cmd/structcheck", - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + PartitionStrategy: partitionPathsAsPackages, defaultEnabled: true, }, "test": { - Name: "test", Command: `go test`, Pattern: `^--- FAIL: .*$\s+(?P.*?\.go):(?P\d+): (?P.*)$`, - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + PartitionStrategy: partitionPathsAsPackages, }, "testify": { - Name: "testify", Command: `go test`, Pattern: `Location:\s+(?P.*?\.go):(?P\d+)$\s+Error:\s+(?P[^\n]+)`, - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + PartitionStrategy: partitionPathsAsPackages, }, "unconvert": { - Name: "unconvert", Command: `unconvert`, Pattern: `PATH:LINE:COL:MESSAGE`, InstallFrom: "github.com/mdempsky/unconvert", - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + PartitionStrategy: partitionPathsAsPackages, defaultEnabled: true, }, "unparam": { - Name: "unparam", - Command: `unparam`, + Command: `unparam {not_tests=-tests=false}`, Pattern: `PATH:LINE:COL:MESSAGE`, - InstallFrom: "github.com/mvdan/unparam", - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + InstallFrom: "mvdan.cc/unparam", + PartitionStrategy: partitionPathsAsPackages, }, "unused": { - Name: "unused", Command: `unused`, Pattern: `PATH:LINE:COL:MESSAGE`, InstallFrom: "honnef.co/go/tools/cmd/unused", - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + PartitionStrategy: partitionPathsAsPackages, }, "varcheck": { - Name: "varcheck", Command: `varcheck`, Pattern: `^(?:[^:]+: )?(?P.*?\.go):(?P\d+):(?P\d+):\s*(?P.*)$`, InstallFrom: "github.com/opennota/check/cmd/varcheck", - PartitionStrategy: partitionToMaxArgSizeWithPackagePaths, + PartitionStrategy: partitionPathsAsPackages, defaultEnabled: true, }, "vet": { - Name: "vet", - Command: `go tool vet`, + Command: `govet --no-recurse`, Pattern: vetPattern, - PartitionStrategy: partitionToPackageFileGlobs, + InstallFrom: "github.com/dnephin/govet", + PartitionStrategy: partitionPathsAsDirectories, defaultEnabled: true, IsFast: true, }, "vetshadow": { - Name: "vetshadow", - Command: `go tool vet --shadow`, + Command: `govet --no-recurse --shadow`, Pattern: vetPattern, - PartitionStrategy: partitionToPackageFileGlobs, + PartitionStrategy: partitionPathsAsDirectories, defaultEnabled: true, IsFast: true, }, diff --git a/vendor/src/github.com/alecthomas/gometalinter/linters_test.go b/vendor/src/github.com/alecthomas/gometalinter/linters_test.go index 1550a9fe2..1a66db001 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/linters_test.go +++ b/vendor/src/github.com/alecthomas/gometalinter/linters_test.go @@ -1,6 +1,8 @@ package main import ( + "reflect" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -12,17 +14,48 @@ func TestNewLinterWithCustomLinter(t *testing.T) { Command: "/usr/bin/custom", Pattern: "path", } - linter, err := NewLinter(config) + linter, err := NewLinter("thename", config) require.NoError(t, err) - assert.NotNil(t, linter.LinterConfig.PartitionStrategy) + assert.Equal(t, functionName(partitionPathsAsDirectories), functionName(linter.LinterConfig.PartitionStrategy)) + assert.Equal(t, "(?m:path)", linter.regex.String()) + assert.Equal(t, "thename", linter.Name) + assert.Equal(t, config.Command, linter.Command) } func TestGetLinterByName(t *testing.T) { config := LinterConfig{ - Command: "aligncheck", - Pattern: "path", + Command: "maligned", + Pattern: "path", + InstallFrom: "./install/path", + PartitionStrategy: partitionPathsAsDirectories, + IsFast: true, } overrideConfig := getLinterByName(config.Command, config) - require.Equal(t, config.Command, overrideConfig.Command) - require.Equal(t, config.Pattern, overrideConfig.Pattern) + assert.Equal(t, config.Command, overrideConfig.Command) + assert.Equal(t, config.Pattern, overrideConfig.Pattern) + assert.Equal(t, config.InstallFrom, overrideConfig.InstallFrom) + assert.Equal(t, functionName(config.PartitionStrategy), functionName(overrideConfig.PartitionStrategy)) + assert.Equal(t, config.IsFast, overrideConfig.IsFast) +} + +func TestValidateLinters(t *testing.T) { + originalConfig := *config + defer func() { config = &originalConfig }() + + config = &Config{ + Enable: []string{"_dummylinter_"}, + } + + err := validateLinters(lintersFromConfig(config), config) + require.Error(t, err, "expected unknown linter error for _dummylinter_") + + config = &Config{ + Enable: defaultEnabled(), + } + err = validateLinters(lintersFromConfig(config), config) + require.NoError(t, err) +} + +func functionName(i interface{}) string { + return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() } diff --git a/vendor/src/github.com/alecthomas/gometalinter/main.go b/vendor/src/github.com/alecthomas/gometalinter/main.go index 6aa60dc32..4770d95d1 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/main.go +++ b/vendor/src/github.com/alecthomas/gometalinter/main.go @@ -14,7 +14,7 @@ import ( "text/template" "time" - "gopkg.in/alecthomas/kingpin.v3-unstable" + kingpin "gopkg.in/alecthomas/kingpin.v3-unstable" ) var ( @@ -51,16 +51,17 @@ func setupFlags(app *kingpin.Application) { app.Flag("line-length", "Report lines longer than N (using lll).").PlaceHolder("80").IntVar(&config.LineLength) app.Flag("min-confidence", "Minimum confidence interval to pass to golint.").PlaceHolder(".80").FloatVar(&config.MinConfidence) app.Flag("min-occurrences", "Minimum occurrences to pass to goconst.").PlaceHolder("3").IntVar(&config.MinOccurrences) - app.Flag("min-const-length", "Minimumum constant length.").PlaceHolder("3").IntVar(&config.MinConstLength) + app.Flag("min-const-length", "Minimum constant length.").PlaceHolder("3").IntVar(&config.MinConstLength) app.Flag("dupl-threshold", "Minimum token sequence as a clone for dupl.").PlaceHolder("50").IntVar(&config.DuplThreshold) app.Flag("sort", fmt.Sprintf("Sort output by any of %s.", strings.Join(sortKeys, ", "))).PlaceHolder("none").EnumsVar(&config.Sort, sortKeys...) - app.Flag("tests", "Include test files for linters that support this option").Short('t').BoolVar(&config.Test) + app.Flag("tests", "Include test files for linters that support this option.").Short('t').BoolVar(&config.Test) app.Flag("deadline", "Cancel linters if they have not completed within this duration.").PlaceHolder("30s").DurationVar((*time.Duration)(&config.Deadline)) app.Flag("errors", "Only show errors.").BoolVar(&config.Errors) app.Flag("json", "Generate structured JSON rather than standard line-based output.").BoolVar(&config.JSON) app.Flag("checkstyle", "Generate checkstyle XML rather than standard line-based output.").BoolVar(&config.Checkstyle) app.Flag("enable-gc", "Enable GC for linters (useful on large repositories).").BoolVar(&config.EnableGC) app.Flag("aggregate", "Aggregate issues reported by several linters.").BoolVar(&config.Aggregate) + app.Flag("warn-unmatched-nolint", "Warn if a nolint directive is not matched with an issue.").BoolVar(&config.WarnUnmatchedDirective) app.GetFlag("help").Short('h') } @@ -200,6 +201,9 @@ Severity override map (default is "warning"): paths := resolvePaths(*pathsArg, config.Skip) linters := lintersFromConfig(config) + err := validateLinters(linters, config) + kingpin.FatalIfError(err, "") + issues, errch := runLinters(linters, paths, config.Concurrency, exclude, include) status := 0 if config.JSON { @@ -222,7 +226,7 @@ Severity override map (default is "warning"): func processConfig(config *Config) (include *regexp.Regexp, exclude *regexp.Regexp) { tmpl, err := template.New("output").Parse(config.Format) kingpin.FatalIfError(err, "invalid format %q", config.Format) - formatTemplate = tmpl + config.formatTemplate = tmpl // Linters are by their very nature, short lived, so disable GC. // Reduced (user) linting time on kingpin from 0.97s to 0.64s. diff --git a/vendor/src/github.com/alecthomas/gometalinter/partition.go b/vendor/src/github.com/alecthomas/gometalinter/partition.go index 708beb8fc..a4a8ce5a5 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/partition.go +++ b/vendor/src/github.com/alecthomas/gometalinter/partition.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "path/filepath" ) @@ -10,6 +11,29 @@ const MaxCommandBytes = 32000 type partitionStrategy func([]string, []string) ([][]string, error) +func (ps *partitionStrategy) UnmarshalJSON(raw []byte) error { + var strategyName string + if err := json.Unmarshal(raw, &strategyName); err != nil { + return err + } + + switch strategyName { + case "directories": + *ps = partitionPathsAsDirectories + case "files": + *ps = partitionPathsAsFiles + case "packages": + *ps = partitionPathsAsPackages + case "files-by-package": + *ps = partitionPathsAsFilesGroupedByPackage + case "single-directory": + *ps = partitionPathsByDirectory + default: + return fmt.Errorf("unknown parition strategy %s", strategyName) + } + return nil +} + func pathsToFileGlobs(paths []string) ([]string, error) { filePaths := []string{} for _, dir := range paths { @@ -22,7 +46,7 @@ func pathsToFileGlobs(paths []string) ([]string, error) { return filePaths, nil } -func partitionToMaxArgSize(cmdArgs []string, paths []string) ([][]string, error) { +func partitionPathsAsDirectories(cmdArgs []string, paths []string) ([][]string, error) { return partitionToMaxSize(cmdArgs, paths, MaxCommandBytes), nil } @@ -72,15 +96,15 @@ func (p *sizePartitioner) end() [][]string { return p.parts } -func partitionToMaxArgSizeWithFileGlobs(cmdArgs []string, paths []string) ([][]string, error) { +func partitionPathsAsFiles(cmdArgs []string, paths []string) ([][]string, error) { filePaths, err := pathsToFileGlobs(paths) if err != nil || len(filePaths) == 0 { return nil, err } - return partitionToMaxArgSize(cmdArgs, filePaths) + return partitionPathsAsDirectories(cmdArgs, filePaths) } -func partitionToPackageFileGlobs(cmdArgs []string, paths []string) ([][]string, error) { +func partitionPathsAsFilesGroupedByPackage(cmdArgs []string, paths []string) ([][]string, error) { parts := [][]string{} for _, path := range paths { filePaths, err := pathsToFileGlobs([]string{path}) @@ -95,12 +119,12 @@ func partitionToPackageFileGlobs(cmdArgs []string, paths []string) ([][]string, return parts, nil } -func partitionToMaxArgSizeWithPackagePaths(cmdArgs []string, paths []string) ([][]string, error) { +func partitionPathsAsPackages(cmdArgs []string, paths []string) ([][]string, error) { packagePaths, err := pathsToPackagePaths(paths) if err != nil || len(packagePaths) == 0 { return nil, err } - return partitionToMaxArgSize(cmdArgs, packagePaths) + return partitionPathsAsDirectories(cmdArgs, packagePaths) } func pathsToPackagePaths(paths []string) ([]string, error) { @@ -129,3 +153,11 @@ func packageNameFromPath(path string) (string, error) { } return "", fmt.Errorf("%s not in GOPATH", path) } + +func partitionPathsByDirectory(cmdArgs []string, paths []string) ([][]string, error) { + parts := [][]string{} + for _, path := range paths { + parts = append(parts, append(cmdArgs, path)) + } + return parts, nil +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/partition_test.go b/vendor/src/github.com/alecthomas/gometalinter/partition_test.go index 07ac305cf..e146486b3 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/partition_test.go +++ b/vendor/src/github.com/alecthomas/gometalinter/partition_test.go @@ -38,7 +38,7 @@ func TestPartitionToPackageFileGlobs(t *testing.T) { mkGoFile(t, dir, "other.go") } - parts, err := partitionToPackageFileGlobs(cmdArgs, paths) + parts, err := partitionPathsAsFilesGroupedByPackage(cmdArgs, paths) require.NoError(t, err) expected := [][]string{ append(cmdArgs, packagePaths(paths[0], "file.go", "other.go")...), @@ -62,7 +62,7 @@ func TestPartitionToPackageFileGlobsNoFiles(t *testing.T) { cmdArgs := []string{"/usr/bin/foo", "-c"} paths := []string{filepath.Join(tmpdir, "one"), filepath.Join(tmpdir, "two")} - parts, err := partitionToPackageFileGlobs(cmdArgs, paths) + parts, err := partitionPathsAsFilesGroupedByPackage(cmdArgs, paths) require.NoError(t, err) assert.Len(t, parts, 0) } @@ -74,7 +74,7 @@ func TestPartitionToMaxArgSizeWithFileGlobsNoFiles(t *testing.T) { cmdArgs := []string{"/usr/bin/foo", "-c"} paths := []string{filepath.Join(tmpdir, "one"), filepath.Join(tmpdir, "two")} - parts, err := partitionToMaxArgSizeWithFileGlobs(cmdArgs, paths) + parts, err := partitionPathsAsFiles(cmdArgs, paths) require.NoError(t, err) assert.Len(t, parts, 0) } @@ -97,3 +97,18 @@ func fakeGoPath(t *testing.T, path string) func() { require.NoError(t, os.Setenv("GOPATH", path)) return func() { require.NoError(t, os.Setenv("GOPATH", oldpath)) } } + +func TestPartitionPathsByDirectory(t *testing.T) { + cmdArgs := []string{"/usr/bin/foo", "-c"} + paths := []string{"one", "two", "three"} + + parts, err := partitionPathsByDirectory(cmdArgs, paths) + require.NoError(t, err) + expected := [][]string{ + append(cmdArgs, "one"), + append(cmdArgs, "two"), + append(cmdArgs, "three"), + } + assert.Equal(t, expected, parts) + +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/aligncheck_test.go b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/aligncheck_test.go deleted file mode 100644 index 90a30ac13..000000000 --- a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/aligncheck_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package regressiontests - -import "testing" - -func TestAlignCheck(t *testing.T) { - t.Parallel() - source := `package test - -type unaligned struct { - a uint16 - b uint64 - c uint16 - -} -` - expected := Issues{ - {Linter: "aligncheck", Severity: "warning", Path: "test.go", Line: 3, Col: 6, Message: "struct unaligned could have size 16 (currently 24)"}, - } - ExpectIssues(t, "aligncheck", source, expected) -} diff --git a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/dupl_test.go b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/dupl_test.go index a9e71d1af..f8a92104b 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/dupl_test.go +++ b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/dupl_test.go @@ -40,8 +40,8 @@ func two() string { ` expected := Issues{ - {Linter: "dupl", Severity: "warning", Path: "test.go", Line: 19, Col: 0, Message: "duplicate of test.go:3-17"}, {Linter: "dupl", Severity: "warning", Path: "test.go", Line: 3, Col: 0, Message: "duplicate of test.go:19-33"}, + {Linter: "dupl", Severity: "warning", Path: "test.go", Line: 19, Col: 0, Message: "duplicate of test.go:3-17"}, } ExpectIssues(t, "dupl", source, expected) } diff --git a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/gosimple_test.go b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/gosimple_test.go index bea7c5654..623e87368 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/gosimple_test.go +++ b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/gosimple_test.go @@ -22,9 +22,9 @@ func a(ok bool, ch chan bool) { } ` expected := Issues{ + {Linter: "gosimple", Severity: "warning", Path: "test.go", Line: 4, Col: 2, Message: "should use a simple channel send/receive instead of select with a single case (S1000)"}, {Linter: "gosimple", Severity: "warning", Path: "test.go", Line: 8, Col: 2, Message: "should use for range instead of for { select {} } (S1000)"}, {Linter: "gosimple", Severity: "warning", Path: "test.go", Line: 14, Col: 5, Message: "should omit comparison to bool constant, can be simplified to ok (S1002)"}, - {Linter: "gosimple", Severity: "warning", Path: "test.go", Line: 4, Col: 2, Message: "should use a simple channel send/receive instead of select with a single case (S1000)"}, } ExpectIssues(t, "gosimple", source, expected) } diff --git a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/gotype_test.go b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/gotype_test.go index c18806f88..ce6e7ff39 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/gotype_test.go +++ b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/gotype_test.go @@ -1,17 +1,55 @@ package regressiontests -import "testing" +import ( + "fmt" + "testing" + + "github.com/gotestyourself/gotestyourself/fs" + "github.com/stretchr/testify/assert" +) func TestGoType(t *testing.T) { t.Parallel() - source := `package test -func test() { + dir := fs.NewDir(t, "test-gotype", + fs.WithFile("file.go", goTypeFile("root")), + fs.WithDir("sub", + fs.WithFile("file.go", goTypeFile("sub"))), + fs.WithDir("excluded", + fs.WithFile("file.go", goTypeFile("excluded")))) + defer dir.Remove() + + expected := Issues{ + {Linter: "gotype", Severity: "error", Path: "file.go", Line: 4, Col: 6, Message: "foo declared but not used"}, + {Linter: "gotype", Severity: "error", Path: "sub/file.go", Line: 4, Col: 6, Message: "foo declared but not used"}, + } + actual := RunLinter(t, "gotype", dir.Path(), "--skip=excluded") + assert.Equal(t, expected, actual) +} + +func TestGoTypeWithMultiPackageDirectoryTest(t *testing.T) { + t.Parallel() + + dir := fs.NewDir(t, "test-gotype", + fs.WithFile("file.go", goTypeFile("root")), + fs.WithFile("file_test.go", goTypeFile("root_test"))) + defer dir.Remove() + + expected := Issues{ + {Linter: "gotype", Severity: "error", Path: "file.go", Line: 4, Col: 6, Message: "foo declared but not used"}, + {Linter: "gotypex", Severity: "error", Path: "file_test.go", Line: 4, Col: 6, Message: "foo declared but not used"}, + } + actual := RunLinter(t, "gotype", dir.Path()) + actual = append(actual, RunLinter(t, "gotypex", dir.Path())...) + assert.Equal(t, expected, actual) +} + + +func goTypeFile(pkg string) string { + return fmt.Sprintf(`package %s + +func badFunction() { var foo string } -` - expected := Issues{ - {Linter: "gotype", Severity: "error", Path: "test.go", Line: 4, Col: 6, Message: "foo declared but not used"}, - } - ExpectIssues(t, "gotype", source, expected) + `, pkg) } diff --git a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/maligned_test.go b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/maligned_test.go new file mode 100644 index 000000000..fe7cc52ed --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/maligned_test.go @@ -0,0 +1,20 @@ +package regressiontests + +import "testing" + +func TestMaligned(t *testing.T) { + t.Parallel() + source := `package test + +type unaligned struct { + a uint16 + b uint64 + c uint16 + +} +` + expected := Issues{ + {Linter: "maligned", Severity: "warning", Path: "test.go", Line: 3, Col: 16, Message: "struct of size 24 could be 16"}, + } + ExpectIssues(t, "maligned", source, expected) +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/nakedret_test.go b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/nakedret_test.go new file mode 100644 index 000000000..d642edbe2 --- /dev/null +++ b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/nakedret_test.go @@ -0,0 +1,29 @@ +package regressiontests + +import "testing" + +func TestNakedret(t *testing.T) { + t.Parallel() + source := `package test + +func shortFunc() (r uint32) { + r = r + r + return +} + +func longFunc() (r uint32) { + r = r + r + r = r - r + r = r * r + r = r / r + r = r % r + r = r^r + r = r&r + return +} +` + expected := Issues{ + {Linter: "nakedret", Severity: "warning", Path: "test.go", Line: 16, Message: "longFunc naked returns on 9 line function "}, + } + ExpectIssues(t, "nakedret", source, expected) +} diff --git a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/support.go b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/support.go index 243e57c09..2563b8b65 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/support.go +++ b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/support.go @@ -8,10 +8,10 @@ import ( "os" "os/exec" "path/filepath" - "sort" "strings" "testing" + "github.com/gotestyourself/gotestyourself/fs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -35,10 +35,6 @@ func (i *Issue) String() string { type Issues []Issue -func (e Issues) Len() int { return len(e) } -func (e Issues) Swap(i, j int) { e[i], e[j] = e[j], e[i] } -func (e Issues) Less(i, j int) bool { return e[i].String() < e[j].String() } - // ExpectIssues runs gometalinter and expects it to generate exactly the // issues provided. func ExpectIssues(t *testing.T, linter string, source string, expected Issues, extraFlags ...string) { @@ -51,50 +47,60 @@ func ExpectIssues(t *testing.T, linter string, source string, expected Issues, e err = ioutil.WriteFile(testFile, []byte(source), 0644) require.NoError(t, err) - // Run gometalinter. + actual := RunLinter(t, linter, dir, extraFlags...) + assert.Equal(t, expected, actual) +} + +// RunLinter runs the gometalinter as a binary against the files at path and +// returns the issues it encountered +func RunLinter(t *testing.T, linter string, path string, extraFlags ...string) Issues { binary, cleanup := buildBinary(t) defer cleanup() - args := []string{"-d", "--disable-all", "--enable", linter, "--json", dir} + + args := []string{ + "-d", "--disable-all", "--enable", linter, "--json", + "--sort=path", "--sort=line", "--sort=column", "--sort=message", + "./...", + } args = append(args, extraFlags...) cmd := exec.Command(binary, args...) + cmd.Dir = path + errBuffer := new(bytes.Buffer) cmd.Stderr = errBuffer - require.NoError(t, err) output, _ := cmd.Output() + var actual Issues - err = json.Unmarshal(output, &actual) + err := json.Unmarshal(output, &actual) if !assert.NoError(t, err) { fmt.Printf("Stderr: %s\n", errBuffer) fmt.Printf("Output: %s\n", output) - return - } - - // Remove output from other linters. - actualForLinter := Issues{} - for _, issue := range actual { - if issue.Linter == linter || linter == "" { - // Normalise path. - issue.Path = "test.go" - issue.Message = strings.Replace(issue.Message, testFile, "test.go", -1) - issue.Message = strings.Replace(issue.Message, dir, "", -1) - actualForLinter = append(actualForLinter, issue) - } - } - sort.Sort(expected) - sort.Sort(actualForLinter) - - if !assert.Equal(t, expected, actualForLinter) { - fmt.Printf("Stderr: %s\n", errBuffer) - fmt.Printf("Output: %s\n", output) + return nil } + return filterIssues(actual, linter, path) } func buildBinary(t *testing.T) (string, func()) { - tmpdir, err := ioutil.TempDir("", "regression-test") - require.NoError(t, err) - path := filepath.Join(tmpdir, "binary") + tmpdir := fs.NewDir(t, "regression-test-binary") + path := tmpdir.Join("gometalinter") cmd := exec.Command("go", "build", "-o", path, "..") require.NoError(t, cmd.Run()) - return path, func() { os.RemoveAll(tmpdir) } + return path, tmpdir.Remove +} + +// filterIssues to just the issues relevant for the current linter and normalize +// the error message by removing the directory part of the path from both Path +// and Message +func filterIssues(issues Issues, linterName string, dir string) Issues { + filtered := Issues{} + for _, issue := range issues { + if issue.Linter == linterName || linterName == "" { + issue.Path = strings.Replace(issue.Path, dir+string(os.PathSeparator), "", -1) + issue.Message = strings.Replace(issue.Message, dir+string(os.PathSeparator), "", -1) + issue.Message = strings.Replace(issue.Message, dir, "", -1) + filtered = append(filtered, issue) + } + } + return filtered } diff --git a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/unparam_test.go b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/unparam_test.go index 212f8ae4c..0596d2bd8 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/unparam_test.go +++ b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/unparam_test.go @@ -1,6 +1,8 @@ package regressiontests -import "testing" +import ( + "testing" +) func TestUnparam(t *testing.T) { t.Parallel() @@ -133,9 +135,6 @@ func AsSliceElem(f FooType) []int { var SliceElems = []func(FooType) []int{AsSliceElem} ` expected := Issues{ - Issue{Linter: "unparam", Severity: "warning", Path: "test.go", Line: 103, Col: 16, Message: "parameter a is unused"}, - Issue{Linter: "unparam", Severity: "warning", Path: "test.go", Line: 103, Col: 27, Message: "parameter r is unused"}, - Issue{Linter: "unparam", Severity: "warning", Path: "test.go", Line: 123, Col: 18, Message: "parameter f is unused"}, Issue{Linter: "unparam", Severity: "warning", Path: "test.go", Line: 13, Col: 19, Message: "parameter b is unused"}, Issue{Linter: "unparam", Severity: "warning", Path: "test.go", Line: 20, Col: 20, Message: "parameter f is unused"}, Issue{Linter: "unparam", Severity: "warning", Path: "test.go", Line: 34, Col: 37, Message: "parameter code is unused"}, @@ -146,6 +145,9 @@ var SliceElems = []func(FooType) []int{AsSliceElem} ` Issue{Linter: "unparam", Severity: "warning", Path: "test.go", Line: 85, Col: 25, Message: "parameter s is unused"}, Issue{Linter: "unparam", Severity: "warning", Path: "test.go", Line: 95, Col: 15, Message: "parameter a is unused"}, Issue{Linter: "unparam", Severity: "warning", Path: "test.go", Line: 95, Col: 26, Message: "parameter b is unused"}, + Issue{Linter: "unparam", Severity: "warning", Path: "test.go", Line: 103, Col: 16, Message: "parameter a is unused"}, + Issue{Linter: "unparam", Severity: "warning", Path: "test.go", Line: 103, Col: 27, Message: "parameter r is unused"}, + Issue{Linter: "unparam", Severity: "warning", Path: "test.go", Line: 123, Col: 18, Message: "parameter f is unused"}, } ExpectIssues(t, "unparam", source, expected) } diff --git a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/vet_test.go b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/vet_test.go index 3e610bdc5..733a5f621 100644 --- a/vendor/src/github.com/alecthomas/gometalinter/regressiontests/vet_test.go +++ b/vendor/src/github.com/alecthomas/gometalinter/regressiontests/vet_test.go @@ -1,20 +1,55 @@ package regressiontests -import "testing" +import ( + "testing" + + "github.com/gotestyourself/gotestyourself/fs" + "github.com/stretchr/testify/assert" +) func TestVet(t *testing.T) { t.Parallel() + + dir := fs.NewDir(t, "test-vet", + fs.WithFile("file.go", vetFile("root")), + fs.WithFile("file_test.go", vetExternalPackageFile("root_test")), + fs.WithDir("sub", + fs.WithFile("file.go", vetFile("sub"))), + fs.WithDir("excluded", + fs.WithFile("file.go", vetFile("excluded")))) + defer dir.Remove() + expected := Issues{ - {Linter: "vet", Severity: "error", Path: "test.go", Line: 7, Col: 0, Message: "missing argument for Printf(\"%d\"): format reads arg 1, have only 0 args"}, - {Linter: "vet", Severity: "error", Path: "test.go", Line: 7, Col: 0, Message: "unreachable code"}, + {Linter: "vet", Severity: "error", Path: "file.go", Line: 7, Col: 0, Message: "missing argument for Printf(\"%d\"): format reads arg 1, have only 0 args"}, + {Linter: "vet", Severity: "error", Path: "file.go", Line: 7, Col: 0, Message: "unreachable code"}, + {Linter: "vet", Severity: "error", Path: "file_test.go", Line: 7, Col: 0, Message: "unreachable code"}, + {Linter: "vet", Severity: "error", Path: "sub/file.go", Line: 7, Col: 0, Message: "missing argument for Printf(\"%d\"): format reads arg 1, have only 0 args"}, + {Linter: "vet", Severity: "error", Path: "sub/file.go", Line: 7, Col: 0, Message: "unreachable code"}, } - ExpectIssues(t, "vet", `package main + actual := RunLinter(t, "vet", dir.Path(), "--skip=excluded") + assert.Equal(t, expected, actual) +} + +func vetFile(pkg string) string { + return `package ` + pkg + ` import "fmt" -func main() { +func Something() { return fmt.Printf("%d") } -`, expected) +` +} + +func vetExternalPackageFile(pkg string) string { + return `package ` + pkg + ` + +import "fmt" + +func ExampleSomething() { + return + root.Something() +} +` } From ff8b229192cb52d7d84b37a375b470d6ddc3d3f0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Nov 2017 11:12:21 +0000 Subject: [PATCH 18/29] Remove vendor/bin from PATH in script (#335) We should no longer be building anything into vendor/bin in the scripts, so adding it to the path can lead to all sorts of confusion if old binaries are there. --- scripts/build-test-lint.sh | 2 +- scripts/find-lint.sh | 2 +- scripts/travis-test.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build-test-lint.sh b/scripts/build-test-lint.sh index 16078173d..f283017fd 100755 --- a/scripts/build-test-lint.sh +++ b/scripts/build-test-lint.sh @@ -5,7 +5,7 @@ set -eu export GOPATH="$(pwd):$(pwd)/vendor" -export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin" +export PATH="$PATH:$(pwd)/bin" echo "Checking that it builds" gb build diff --git a/scripts/find-lint.sh b/scripts/find-lint.sh index e2df519c5..c7b66b287 100755 --- a/scripts/find-lint.sh +++ b/scripts/find-lint.sh @@ -14,7 +14,7 @@ set -eu export GOPATH="$(pwd):$(pwd)/vendor" -export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin" +export PATH="$PATH:$(pwd)/bin" args="" if [ ${1:-""} = "fast" ] diff --git a/scripts/travis-test.sh b/scripts/travis-test.sh index e472760a6..ace9a6c0e 100755 --- a/scripts/travis-test.sh +++ b/scripts/travis-test.sh @@ -13,7 +13,7 @@ export GOGC=400 export DENDRITE_LINT_DISABLE_GC=1 export GOPATH="$(pwd):$(pwd)/vendor" -export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin" +export PATH="$PATH:$(pwd)/bin" if [ "${TEST_SUITE:-lint}" == "lint" ]; then ./scripts/find-lint.sh From 139fae988ecaa6af877e16995ed3ad0f5618f5ea Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Nov 2017 11:13:09 +0000 Subject: [PATCH 19/29] Fix linting errors in go1.9.1 (#336) Fix linting errors in go1.9.1 --- .../dendrite/clientapi/consumers/roomserver.go | 6 +----- .../dendrite/clientapi/threepid/invites.go | 6 +----- .../matrix-org/dendrite/common/test/config.go | 5 +---- .../dendrite/federationapi/routing/profile.go | 8 ++++---- .../dendrite/federationapi/routing/send.go | 11 ++--------- .../dendrite/federationsender/storage/storage.go | 6 +----- .../matrix-org/dendrite/mediaapi/routing/upload.go | 10 +++------- .../matrix-org/dendrite/mediaapi/storage/sql.go | 10 ++++------ .../matrix-org/dendrite/roomserver/alias/alias.go | 12 ++---------- .../matrix-org/dendrite/roomserver/input/events.go | 6 +----- .../dendrite/roomserver/input/latest_events.go | 6 +----- 11 files changed, 21 insertions(+), 65 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/clientapi/consumers/roomserver.go b/src/github.com/matrix-org/dendrite/clientapi/consumers/roomserver.go index e915ddd25..0ee7c6bf0 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/consumers/roomserver.go +++ b/src/github.com/matrix-org/dendrite/clientapi/consumers/roomserver.go @@ -96,11 +96,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { return err } - if err := s.db.UpdateMemberships(context.TODO(), events, output.NewRoomEvent.RemovesStateEventIDs); err != nil { - return err - } - - return nil + return s.db.UpdateMemberships(context.TODO(), events, output.NewRoomEvent.RemovesStateEventIDs) } // lookupStateEvents looks up the state events that are added by a new event. diff --git a/src/github.com/matrix-org/dendrite/clientapi/threepid/invites.go b/src/github.com/matrix-org/dendrite/clientapi/threepid/invites.go index 836591829..ea998e34a 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/threepid/invites.go +++ b/src/github.com/matrix-org/dendrite/clientapi/threepid/invites.go @@ -355,9 +355,5 @@ func emit3PIDInviteEvent( return err } - if err := producer.SendEvents(ctx, []gomatrixserverlib.Event{*event}, cfg.Matrix.ServerName); err != nil { - return err - } - - return nil + return producer.SendEvents(ctx, []gomatrixserverlib.Event{*event}, cfg.Matrix.ServerName) } diff --git a/src/github.com/matrix-org/dendrite/common/test/config.go b/src/github.com/matrix-org/dendrite/common/test/config.go index ad878c5b8..b9aec5d74 100644 --- a/src/github.com/matrix-org/dendrite/common/test/config.go +++ b/src/github.com/matrix-org/dendrite/common/test/config.go @@ -113,10 +113,7 @@ func WriteConfig(cfg *config.Dendrite, configDir string) error { if err != nil { return err } - if err = ioutil.WriteFile(filepath.Join(configDir, ConfigFile), data, 0666); err != nil { - return err - } - return nil + return ioutil.WriteFile(filepath.Join(configDir, ConfigFile), data, 0666) } // NewMatrixKey generates a new ed25519 matrix server key and writes it to a file. diff --git a/src/github.com/matrix-org/dendrite/federationapi/routing/profile.go b/src/github.com/matrix-org/dendrite/federationapi/routing/profile.go index 48f473669..715ec4d48 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/routing/profile.go +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/profile.go @@ -63,11 +63,11 @@ func GetProfile( switch field { case "displayname": res = common.DisplayName{ - profile.DisplayName, + DisplayName: profile.DisplayName, } case "avatar_url": res = common.AvatarURL{ - profile.AvatarURL, + AvatarURL: profile.AvatarURL, } default: code = 400 @@ -75,8 +75,8 @@ func GetProfile( } } else { res = common.ProfileResponse{ - profile.AvatarURL, - profile.DisplayName, + AvatarURL: profile.AvatarURL, + DisplayName: profile.DisplayName, } } diff --git a/src/github.com/matrix-org/dendrite/federationapi/routing/send.go b/src/github.com/matrix-org/dendrite/federationapi/routing/send.go index 47c5efd0f..2103c2b8f 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/routing/send.go +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/send.go @@ -170,11 +170,7 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { // TODO: Check that the event is allowed by its auth_events. // pass the event to the roomserver - if err := t.producer.SendEvents(t.context, []gomatrixserverlib.Event{e}, api.DoNotSendToOtherServers); err != nil { - return err - } - - return nil + return t.producer.SendEvents(t.context, []gomatrixserverlib.Event{e}, api.DoNotSendToOtherServers) } func checkAllowedByState(e gomatrixserverlib.Event, stateEvents []gomatrixserverlib.Event) error { @@ -218,8 +214,5 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event) error { return err } // pass the event along with the state to the roomserver - if err := t.producer.SendEventWithState(t.context, state, e); err != nil { - return err - } - return nil + return t.producer.SendEventWithState(t.context, state, e) } diff --git a/src/github.com/matrix-org/dendrite/federationsender/storage/storage.go b/src/github.com/matrix-org/dendrite/federationsender/storage/storage.go index ab97dc449..e84d639d0 100644 --- a/src/github.com/matrix-org/dendrite/federationsender/storage/storage.go +++ b/src/github.com/matrix-org/dendrite/federationsender/storage/storage.go @@ -54,11 +54,7 @@ func (d *Database) prepare() error { return err } - if err = d.PartitionOffsetStatements.Prepare(d.db, "federationsender"); err != nil { - return err - } - - return nil + return d.PartitionOffsetStatements.Prepare(d.db, "federationsender") } // UpdateRoom updates the joined hosts for a room and returns what the joined diff --git a/src/github.com/matrix-org/dendrite/mediaapi/routing/upload.go b/src/github.com/matrix-org/dendrite/mediaapi/routing/upload.go index 64def9eb3..aafc0230a 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/routing/upload.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/routing/upload.go @@ -22,7 +22,6 @@ import ( "net/url" "path" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/fileutils" @@ -31,6 +30,7 @@ import ( "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" ) // uploadRequest metadata included in or derivable from an upload request @@ -161,14 +161,10 @@ func (r *uploadRequest) doUpload( } } - if resErr := r.storeFileAndMetadata( + return r.storeFileAndMetadata( ctx, tmpDir, cfg.Media.AbsBasePath, db, cfg.Media.ThumbnailSizes, activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators, - ); resErr != nil { - return resErr - } - - return nil + ) } // Validate validates the uploadRequest fields diff --git a/src/github.com/matrix-org/dendrite/mediaapi/storage/sql.go b/src/github.com/matrix-org/dendrite/mediaapi/storage/sql.go index 024ab8bb8..1f8c7be30 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/storage/sql.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/storage/sql.go @@ -23,15 +23,13 @@ type statements struct { thumbnail thumbnailStatements } -func (s *statements) prepare(db *sql.DB) error { - var err error - +func (s *statements) prepare(db *sql.DB) (err error) { if err = s.media.prepare(db); err != nil { - return err + return } if err = s.thumbnail.prepare(db); err != nil { - return err + return } - return nil + return } diff --git a/src/github.com/matrix-org/dendrite/roomserver/alias/alias.go b/src/github.com/matrix-org/dendrite/roomserver/alias/alias.go index 5f5436ebf..7f41b4629 100644 --- a/src/github.com/matrix-org/dendrite/roomserver/alias/alias.go +++ b/src/github.com/matrix-org/dendrite/roomserver/alias/alias.go @@ -79,11 +79,7 @@ func (r *RoomserverAliasAPI) SetRoomAlias( // At this point we've already committed the alias to the database so we // shouldn't cancel this request. // TODO: Ensure that we send unsent events when if server restarts. - if err := r.sendUpdatedAliasesEvent(context.TODO(), request.UserID, request.RoomID); err != nil { - return err - } - - return nil + return r.sendUpdatedAliasesEvent(context.TODO(), request.UserID, request.RoomID) } // GetAliasRoomID implements api.RoomserverAliasAPI @@ -123,11 +119,7 @@ func (r *RoomserverAliasAPI) RemoveRoomAlias( // At this point we've already committed the alias to the database so we // shouldn't cancel this request. // TODO: Ensure that we send unsent events when if server restarts. - if err := r.sendUpdatedAliasesEvent(context.TODO(), request.UserID, roomID); err != nil { - return err - } - - return nil + return r.sendUpdatedAliasesEvent(context.TODO(), request.UserID, roomID) } type roomAliasesContent struct { diff --git a/src/github.com/matrix-org/dendrite/roomserver/input/events.go b/src/github.com/matrix-org/dendrite/roomserver/input/events.go index 57b20961d..9032219ee 100644 --- a/src/github.com/matrix-org/dendrite/roomserver/input/events.go +++ b/src/github.com/matrix-org/dendrite/roomserver/input/events.go @@ -129,11 +129,7 @@ func processRoomEvent( } // Update the extremities of the event graph for the room - if err := updateLatestEvents(ctx, db, ow, roomNID, stateAtEvent, event, input.SendAsServer); err != nil { - return err - } - - return nil + return updateLatestEvents(ctx, db, ow, roomNID, stateAtEvent, event, input.SendAsServer) } func processInviteEvent( diff --git a/src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go b/src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go index 30129a626..5767daab8 100644 --- a/src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go +++ b/src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go @@ -162,11 +162,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { return err } - if err = u.updater.MarkEventAsSent(u.stateAtEvent.EventNID); err != nil { - return err - } - - return nil + return u.updater.MarkEventAsSent(u.stateAtEvent.EventNID) } func (u *latestEventsUpdater) latestState() error { From db32692f2e9ded2ab65c8e45673657db7b8984db Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 15 Nov 2017 11:14:21 +0000 Subject: [PATCH 20/29] Just run the linter in the commit hook (#334) The pre-commit hook took 45 seconds to run on my machine, which was more than enough time for me to get distracted by a swordfight in the corridor. Let's just run the linters (which still take 6 seconds). It's not the place of a commit hook to run every test we can think of - that is what CI is for. --- hooks/pre-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/pre-commit b/hooks/pre-commit index ec7e3e1e3..ae96b9ce7 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -2,4 +2,4 @@ set -eu -./scripts/build-test-lint.sh +./scripts/find-lint.sh fast From 7f854224716f1f65ea6e0fabab3e7fe9d51e5491 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Nov 2017 15:42:39 +0000 Subject: [PATCH 21/29] Move /state request handling out of RequestPool (#333) We should probably move the handling out from the syncapi, but that requires the clientapi to stream the current state which it currently doesn't. This at least stops the sync and state handling being done in one file. --- .../cmd/dendrite-monolith-server/main.go | 2 +- .../cmd/dendrite-sync-api-server/main.go | 2 +- .../dendrite/syncapi/routing/routing.go | 9 +- .../dendrite/syncapi/routing/state.go | 118 ++++++++++++++++++ .../dendrite/syncapi/sync/requestpool.go | 91 -------------- 5 files changed, 125 insertions(+), 97 deletions(-) create mode 100644 src/github.com/matrix-org/dendrite/syncapi/routing/state.go diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go index 3b5975763..ea71fed98 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go @@ -330,7 +330,7 @@ func (m *monolith) setupAPIs() { syncapi_routing.Setup(m.api, syncapi_sync.NewRequestPool( m.syncAPIDB, m.syncAPINotifier, m.accountDB, - ), m.deviceDB) + ), m.syncAPIDB, m.deviceDB) federationapi_routing.Setup( m.api, *m.cfg, m.queryAPI, m.roomServerProducer, m.keyRing, m.federation, diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go index fb81b0e3d..e7f83a60d 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go @@ -104,7 +104,7 @@ func main() { log.Info("Starting sync server on ", cfg.Listen.SyncAPI) api := mux.NewRouter() - routing.Setup(api, sync.NewRequestPool(db, n, adb), deviceDB) + routing.Setup(api, sync.NewRequestPool(db, n, adb), db, deviceDB) common.SetupHTTPAPI(http.DefaultServeMux, api) log.Fatal(http.ListenAndServe(string(cfg.Listen.SyncAPI), nil)) diff --git a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go index 15ca267a0..6cdbe992a 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/util" ) @@ -28,7 +29,7 @@ import ( const pathPrefixR0 = "/_matrix/client/r0" // Setup configures the given mux with sync-server listeners -func Setup(apiMux *mux.Router, srp *sync.RequestPool, deviceDB *devices.Database) { +func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatabase, deviceDB *devices.Database) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() r0mux.Handle("/sync", common.MakeAuthAPI("sync", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { @@ -37,16 +38,16 @@ func Setup(apiMux *mux.Router, srp *sync.RequestPool, deviceDB *devices.Database r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars := mux.Vars(req) - return srp.OnIncomingStateRequest(req, vars["roomID"]) + return OnIncomingStateRequest(req, syncDB, vars["roomID"]) })).Methods("GET") r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars := mux.Vars(req) - return srp.OnIncomingStateTypeRequest(req, vars["roomID"], vars["type"], "") + return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], "") })).Methods("GET") r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars := mux.Vars(req) - return srp.OnIncomingStateTypeRequest(req, vars["roomID"], vars["type"], vars["stateKey"]) + return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], vars["stateKey"]) })).Methods("GET") } diff --git a/src/github.com/matrix-org/dendrite/syncapi/routing/state.go b/src/github.com/matrix-org/dendrite/syncapi/routing/state.go new file mode 100644 index 000000000..6c825fce8 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/syncapi/routing/state.go @@ -0,0 +1,118 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "encoding/json" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" +) + +type stateEventInStateResp struct { + gomatrixserverlib.ClientEvent + PrevContent json.RawMessage `json:"prev_content,omitempty"` + ReplacesState string `json:"replaces_state,omitempty"` +} + +// OnIncomingStateRequest is called when a client makes a /rooms/{roomID}/state +// request. It will fetch all the state events from the specified room and will +// append the necessary keys to them if applicable before returning them. +// Returns an error if something went wrong in the process. +// TODO: Check if the user is in the room. If not, check if the room's history +// is publicly visible. Current behaviour is returning an empty array if the +// user cannot see the room's history. +func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatabase, roomID string) util.JSONResponse { + // TODO(#287): Auth request and handle the case where the user has left (where + // we should return the state at the poin they left) + + stateEvents, err := db.GetStateEventsForRoom(req.Context(), roomID) + if err != nil { + return httputil.LogThenError(req, err) + } + + resp := []stateEventInStateResp{} + // Fill the prev_content and replaces_state keys if necessary + for _, event := range stateEvents { + stateEvent := stateEventInStateResp{ + ClientEvent: gomatrixserverlib.ToClientEvent(event, gomatrixserverlib.FormatAll), + } + var prevEventRef types.PrevEventRef + if len(event.Unsigned()) > 0 { + if err := json.Unmarshal(event.Unsigned(), &prevEventRef); err != nil { + return httputil.LogThenError(req, err) + } + // Fills the previous state event ID if the state event replaces another + // state event + if len(prevEventRef.ReplacesState) > 0 { + stateEvent.ReplacesState = prevEventRef.ReplacesState + } + // Fill the previous event if the state event references a previous event + if prevEventRef.PrevContent != nil { + stateEvent.PrevContent = prevEventRef.PrevContent + } + } + + resp = append(resp, stateEvent) + } + + return util.JSONResponse{ + Code: 200, + JSON: resp, + } +} + +// OnIncomingStateTypeRequest is called when a client makes a +// /rooms/{roomID}/state/{type}/{statekey} request. It will look in current +// state to see if there is an event with that type and state key, if there +// is then (by default) we return the content, otherwise a 404. +func OnIncomingStateTypeRequest(req *http.Request, db *storage.SyncServerDatabase, roomID string, evType, stateKey string) util.JSONResponse { + // TODO(#287): Auth request and handle the case where the user has left (where + // we should return the state at the poin they left) + + logger := util.GetLogger(req.Context()) + logger.WithFields(log.Fields{ + "roomID": roomID, + "evType": evType, + "stateKey": stateKey, + }).Info("Fetching state") + + event, err := db.GetStateEvent(req.Context(), roomID, evType, stateKey) + if err != nil { + return httputil.LogThenError(req, err) + } + + if event == nil { + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound("cannot find state"), + } + } + + stateEvent := stateEventInStateResp{ + ClientEvent: gomatrixserverlib.ToClientEvent(*event, gomatrixserverlib.FormatAll), + } + + return util.JSONResponse{ + Code: 200, + JSON: stateEvent.Content, + } +} diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go b/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go index ebfb140d1..3b6775618 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go @@ -15,7 +15,6 @@ package sync import ( - "encoding/json" "net/http" "time" @@ -120,96 +119,6 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype } } -type stateEventInStateResp struct { - gomatrixserverlib.ClientEvent - PrevContent json.RawMessage `json:"prev_content,omitempty"` - ReplacesState string `json:"replaces_state,omitempty"` -} - -// OnIncomingStateRequest is called when a client makes a /rooms/{roomID}/state -// request. It will fetch all the state events from the specified room and will -// append the necessary keys to them if applicable before returning them. -// Returns an error if something went wrong in the process. -// TODO: Check if the user is in the room. If not, check if the room's history -// is publicly visible. Current behaviour is returning an empty array if the -// user cannot see the room's history. -func (rp *RequestPool) OnIncomingStateRequest(req *http.Request, roomID string) util.JSONResponse { - // TODO(#287): Auth request and handle the case where the user has left (where - // we should return the state at the poin they left) - - stateEvents, err := rp.db.GetStateEventsForRoom(req.Context(), roomID) - if err != nil { - return httputil.LogThenError(req, err) - } - - resp := []stateEventInStateResp{} - // Fill the prev_content and replaces_state keys if necessary - for _, event := range stateEvents { - stateEvent := stateEventInStateResp{ - ClientEvent: gomatrixserverlib.ToClientEvent(event, gomatrixserverlib.FormatAll), - } - var prevEventRef types.PrevEventRef - if len(event.Unsigned()) > 0 { - if err := json.Unmarshal(event.Unsigned(), &prevEventRef); err != nil { - return httputil.LogThenError(req, err) - } - // Fills the previous state event ID if the state event replaces another - // state event - if len(prevEventRef.ReplacesState) > 0 { - stateEvent.ReplacesState = prevEventRef.ReplacesState - } - // Fill the previous event if the state event references a previous event - if prevEventRef.PrevContent != nil { - stateEvent.PrevContent = prevEventRef.PrevContent - } - } - - resp = append(resp, stateEvent) - } - - return util.JSONResponse{ - Code: 200, - JSON: resp, - } -} - -// OnIncomingStateTypeRequest is called when a client makes a -// /rooms/{roomID}/state/{type}/{statekey} request. It will look in current -// state to see if there is an event with that type and state key, if there -// is then (by default) we return the content, otherwise a 404. -func (rp *RequestPool) OnIncomingStateTypeRequest(req *http.Request, roomID string, evType, stateKey string) util.JSONResponse { - // TODO(#287): Auth request and handle the case where the user has left (where - // we should return the state at the poin they left) - - logger := util.GetLogger(req.Context()) - logger.WithFields(log.Fields{ - "roomID": roomID, - "evType": evType, - "stateKey": stateKey, - }).Info("Fetching state") - - event, err := rp.db.GetStateEvent(req.Context(), roomID, evType, stateKey) - if err != nil { - return httputil.LogThenError(req, err) - } - - if event == nil { - return util.JSONResponse{ - Code: 404, - JSON: jsonerror.NotFound("cannot find state"), - } - } - - stateEvent := stateEventInStateResp{ - ClientEvent: gomatrixserverlib.ToClientEvent(*event, gomatrixserverlib.FormatAll), - } - - return util.JSONResponse{ - Code: 200, - JSON: stateEvent.Content, - } -} - func (rp *RequestPool) currentSyncForUser(req syncRequest, currentPos types.StreamPosition) (res *types.Response, err error) { // TODO: handle ignored users if req.since == types.StreamPosition(0) { From 4124ce2ac0b846ae4f9cdd75a4efe21bf4fe1c4f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 15 Nov 2017 17:46:16 +0000 Subject: [PATCH 22/29] Store keys rather than json in the keydatabase (#330) * bump gomatrixserverlib (changes to KeyFetcher and KeyDatabase interfaces) * Store keys rather than json in the keydatabase Rather than storing the raw JSON returned from a /keys/v1/query call in the table, store the key itself. This makes keydb.Database implement the updated KeyDatabase interface. --- .../matrix-org/dendrite/common/keydb/keydb.go | 4 +- .../dendrite/common/keydb/server_key_table.go | 60 ++++---- .../dendrite/federationapi/routing/keys.go | 1 - vendor/manifest | 2 +- .../matrix-org/gomatrixserverlib/base64.go | 28 ++-- .../matrix-org/gomatrixserverlib/client.go | 128 ++++++++++++------ .../gomatrixserverlib/federationclient.go | 57 +------- .../matrix-org/gomatrixserverlib/keyring.go | 106 ++++++++++++--- .../gomatrixserverlib/keyring_test.go | 41 ++++-- .../matrix-org/gomatrixserverlib/keys.go | 3 - .../matrix-org/gomatrixserverlib/linter.json | 2 +- 11 files changed, 265 insertions(+), 167 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/common/keydb/keydb.go b/src/github.com/matrix-org/dendrite/common/keydb/keydb.go index 51444ab29..9e59f0cae 100644 --- a/src/github.com/matrix-org/dendrite/common/keydb/keydb.go +++ b/src/github.com/matrix-org/dendrite/common/keydb/keydb.go @@ -48,14 +48,14 @@ func NewDatabase(dataSourceName string) (*Database, error) { func (d *Database) FetchKeys( ctx context.Context, requests map[gomatrixserverlib.PublicKeyRequest]gomatrixserverlib.Timestamp, -) (map[gomatrixserverlib.PublicKeyRequest]gomatrixserverlib.ServerKeys, error) { +) (map[gomatrixserverlib.PublicKeyRequest]gomatrixserverlib.PublicKeyLookupResult, error) { return d.statements.bulkSelectServerKeys(ctx, requests) } // StoreKeys implements gomatrixserverlib.KeyDatabase func (d *Database) StoreKeys( ctx context.Context, - keyMap map[gomatrixserverlib.PublicKeyRequest]gomatrixserverlib.ServerKeys, + keyMap map[gomatrixserverlib.PublicKeyRequest]gomatrixserverlib.PublicKeyLookupResult, ) error { // TODO: Inserting all the keys within a single transaction may // be more efficient since the transaction overhead can be quite diff --git a/src/github.com/matrix-org/dendrite/common/keydb/server_key_table.go b/src/github.com/matrix-org/dendrite/common/keydb/server_key_table.go index 7d9455c12..e3f2d0d03 100644 --- a/src/github.com/matrix-org/dendrite/common/keydb/server_key_table.go +++ b/src/github.com/matrix-org/dendrite/common/keydb/server_key_table.go @@ -17,14 +17,13 @@ package keydb import ( "context" "database/sql" - "encoding/json" "github.com/lib/pq" "github.com/matrix-org/gomatrixserverlib" ) const serverKeysSchema = ` --- A cache of server keys downloaded from remote servers. +-- A cache of signing keys downloaded from remote servers. CREATE TABLE IF NOT EXISTS keydb_server_keys ( -- The name of the matrix server the key is for. server_name TEXT NOT NULL, @@ -33,10 +32,14 @@ CREATE TABLE IF NOT EXISTS keydb_server_keys ( -- Combined server name and key ID separated by the ASCII unit separator -- to make it easier to run bulk queries. server_name_and_key_id TEXT NOT NULL, - -- When the keys are valid until as a millisecond timestamp. + -- When the key is valid until as a millisecond timestamp. + -- 0 if this is an expired key (in which case expired_ts will be non-zero) valid_until_ts BIGINT NOT NULL, - -- The raw JSON for the server key. - server_key_json TEXT NOT NULL, + -- When the key expired as a millisecond timestamp. + -- 0 if this is an active key (in which case valid_until_ts will be non-zero) + expired_ts BIGINT NOT NULL, + -- The base64-encoded public key. + server_key TEXT NOT NULL, CONSTRAINT keydb_server_keys_unique UNIQUE (server_name, server_key_id) ); @@ -44,15 +47,16 @@ CREATE INDEX IF NOT EXISTS keydb_server_name_and_key_id ON keydb_server_keys (se ` const bulkSelectServerKeysSQL = "" + - "SELECT server_name, server_key_id, server_key_json FROM keydb_server_keys" + + "SELECT server_name, server_key_id, valid_until_ts, expired_ts, " + + " server_key FROM keydb_server_keys" + " WHERE server_name_and_key_id = ANY($1)" const upsertServerKeysSQL = "" + "INSERT INTO keydb_server_keys (server_name, server_key_id," + - " server_name_and_key_id, valid_until_ts, server_key_json)" + - " VALUES ($1, $2, $3, $4, $5)" + + " server_name_and_key_id, valid_until_ts, expired_ts, server_key)" + + " VALUES ($1, $2, $3, $4, $5, $6)" + " ON CONFLICT ON CONSTRAINT keydb_server_keys_unique" + - " DO UPDATE SET valid_until_ts = $4, server_key_json = $5" + " DO UPDATE SET valid_until_ts = $4, expired_ts = $5, server_key = $6" type serverKeyStatements struct { bulkSelectServerKeysStmt *sql.Stmt @@ -76,7 +80,7 @@ func (s *serverKeyStatements) prepare(db *sql.DB) (err error) { func (s *serverKeyStatements) bulkSelectServerKeys( ctx context.Context, requests map[gomatrixserverlib.PublicKeyRequest]gomatrixserverlib.Timestamp, -) (map[gomatrixserverlib.PublicKeyRequest]gomatrixserverlib.ServerKeys, error) { +) (map[gomatrixserverlib.PublicKeyRequest]gomatrixserverlib.PublicKeyLookupResult, error) { var nameAndKeyIDs []string for request := range requests { nameAndKeyIDs = append(nameAndKeyIDs, nameAndKeyID(request)) @@ -87,23 +91,30 @@ func (s *serverKeyStatements) bulkSelectServerKeys( return nil, err } defer rows.Close() // nolint: errcheck - results := map[gomatrixserverlib.PublicKeyRequest]gomatrixserverlib.ServerKeys{} + results := map[gomatrixserverlib.PublicKeyRequest]gomatrixserverlib.PublicKeyLookupResult{} for rows.Next() { var serverName string var keyID string - var keyJSON []byte - if err := rows.Scan(&serverName, &keyID, &keyJSON); err != nil { - return nil, err - } - var serverKeys gomatrixserverlib.ServerKeys - if err := json.Unmarshal(keyJSON, &serverKeys); err != nil { + var key string + var validUntilTS int64 + var expiredTS int64 + if err = rows.Scan(&serverName, &keyID, &validUntilTS, &expiredTS, &key); err != nil { return nil, err } r := gomatrixserverlib.PublicKeyRequest{ ServerName: gomatrixserverlib.ServerName(serverName), KeyID: gomatrixserverlib.KeyID(keyID), } - results[r] = serverKeys + vk := gomatrixserverlib.VerifyKey{} + err = vk.Key.Decode(key) + if err != nil { + return nil, err + } + results[r] = gomatrixserverlib.PublicKeyLookupResult{ + VerifyKey: vk, + ValidUntilTS: gomatrixserverlib.Timestamp(validUntilTS), + ExpiredTS: gomatrixserverlib.Timestamp(expiredTS), + } } return results, nil } @@ -111,19 +122,16 @@ func (s *serverKeyStatements) bulkSelectServerKeys( func (s *serverKeyStatements) upsertServerKeys( ctx context.Context, request gomatrixserverlib.PublicKeyRequest, - keys gomatrixserverlib.ServerKeys, + key gomatrixserverlib.PublicKeyLookupResult, ) error { - keyJSON, err := json.Marshal(keys) - if err != nil { - return err - } - _, err = s.upsertServerKeysStmt.ExecContext( + _, err := s.upsertServerKeysStmt.ExecContext( ctx, string(request.ServerName), string(request.KeyID), nameAndKeyID(request), - int64(keys.ValidUntilTS), - keyJSON, + key.ValidUntilTS, + key.ExpiredTS, + key.Key.Encode(), ) return err } diff --git a/src/github.com/matrix-org/dendrite/federationapi/routing/keys.go b/src/github.com/matrix-org/dendrite/federationapi/routing/keys.go index ea44e4c05..b96d8c5c9 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/routing/keys.go +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/keys.go @@ -38,7 +38,6 @@ func localKeys(cfg config.Dendrite, validUntil time.Time) (*gomatrixserverlib.Se var keys gomatrixserverlib.ServerKeys keys.ServerName = cfg.Matrix.ServerName - keys.FromServer = cfg.Matrix.ServerName publicKey := cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey) diff --git a/vendor/manifest b/vendor/manifest index 6ee014b4a..ff61798d5 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -135,7 +135,7 @@ { "importpath": "github.com/matrix-org/gomatrixserverlib", "repository": "https://github.com/matrix-org/gomatrixserverlib", - "revision": "fb17c27f65a0699b0d15f5311a530225b4aea5e0", + "revision": "076933f95312aae3a9476e78d6b4118e1b45d542", "branch": "master" }, { diff --git a/vendor/src/github.com/matrix-org/gomatrixserverlib/base64.go b/vendor/src/github.com/matrix-org/gomatrixserverlib/base64.go index 63ff90bcc..38ae76951 100644 --- a/vendor/src/github.com/matrix-org/gomatrixserverlib/base64.go +++ b/vendor/src/github.com/matrix-org/gomatrixserverlib/base64.go @@ -26,13 +26,31 @@ import ( // When the bytes are unmarshalled from JSON they are decoded from base64. type Base64String []byte +// Encode encodes the bytes as base64 +func (b64 Base64String) Encode() string { + return base64.RawStdEncoding.EncodeToString(b64) +} + +// Decode decodes the given input into this Base64String +func (b64 *Base64String) Decode(str string) error { + // We must check whether the string was encoded in a URL-safe way in order + // to use the appropriate encoding. + var err error + if strings.ContainsAny(str, "-_") { + *b64, err = base64.RawURLEncoding.DecodeString(str) + } else { + *b64, err = base64.RawStdEncoding.DecodeString(str) + } + return err +} + // MarshalJSON encodes the bytes as base64 and then encodes the base64 as a JSON string. // This takes a value receiver so that maps and slices of Base64String encode correctly. func (b64 Base64String) MarshalJSON() ([]byte, error) { // This could be made more efficient by using base64.RawStdEncoding.Encode // to write the base64 directly to the JSON. We don't need to JSON escape // any of the characters used in base64. - return json.Marshal(base64.RawStdEncoding.EncodeToString(b64)) + return json.Marshal(b64.Encode()) } // UnmarshalJSON decodes a JSON string and then decodes the resulting base64. @@ -44,12 +62,6 @@ func (b64 *Base64String) UnmarshalJSON(raw []byte) (err error) { if err = json.Unmarshal(raw, &str); err != nil { return } - // We must check whether the string was encoded in a URL-safe way in order - // to use the appropriate encoding. - if strings.ContainsAny(str, "-_") { - *b64, err = base64.RawURLEncoding.DecodeString(str) - } else { - *b64, err = base64.RawStdEncoding.DecodeString(str) - } + err = b64.Decode(str) return } diff --git a/vendor/src/github.com/matrix-org/gomatrixserverlib/client.go b/vendor/src/github.com/matrix-org/gomatrixserverlib/client.go index 4274169a9..cad361aed 100644 --- a/vendor/src/github.com/matrix-org/gomatrixserverlib/client.go +++ b/vendor/src/github.com/matrix-org/gomatrixserverlib/client.go @@ -26,10 +26,15 @@ import ( "net/http" "net/url" "strings" + "time" + "github.com/matrix-org/gomatrix" "github.com/matrix-org/util" ) +// Default HTTPS request timeout +const requestTimeout time.Duration = time.Duration(30) * time.Second + // A Client makes request to the federation listeners of matrix // homeservers type Client struct { @@ -41,9 +46,16 @@ type UserInfo struct { Sub string `json:"sub"` } -// NewClient makes a new Client +// NewClient makes a new Client (with default timeout) func NewClient() *Client { - return &Client{client: http.Client{Transport: newFederationTripper()}} + return NewClientWithTimeout(requestTimeout) +} + +// NewClientWithTimeout makes a new Client with a specified request timeout +func NewClientWithTimeout(timeout time.Duration) *Client { + return &Client{client: http.Client{ + Transport: newFederationTripper(), + Timeout: timeout}} } type federationTripper struct { @@ -132,7 +144,7 @@ func (fc *Client) LookupUserInfo( } var response *http.Response - response, err = fc.doHTTPRequest(ctx, req) + response, err = fc.DoHTTPRequest(ctx, req) if response != nil { defer response.Body.Close() // nolint: errcheck } @@ -171,10 +183,10 @@ func (fc *Client) LookupUserInfo( // Perspective servers can use that timestamp to determine whether they can // return a cached copy of the keys or whether they will need to retrieve a fresh // copy of the keys. -// Returns the keys or an error if there was a problem talking to the server. -func (fc *Client) LookupServerKeys( // nolint: gocyclo +// Returns the keys returned by the server, or an error if there was a problem talking to the server. +func (fc *Client) LookupServerKeys( ctx context.Context, matrixServer ServerName, keyRequests map[PublicKeyRequest]Timestamp, -) (map[PublicKeyRequest]ServerKeys, error) { +) ([]ServerKeys, error) { url := url.URL{ Scheme: "matrix", Host: string(matrixServer), @@ -203,48 +215,24 @@ func (fc *Client) LookupServerKeys( // nolint: gocyclo return nil, err } + var body struct { + ServerKeyList []ServerKeys `json:"server_keys"` + } + req, err := http.NewRequest("POST", url.String(), bytes.NewBuffer(requestBytes)) if err != nil { return nil, err } req.Header.Add("Content-Type", "application/json") - response, err := fc.doHTTPRequest(ctx, req) - if response != nil { - defer response.Body.Close() // nolint: errcheck - } + err = fc.DoRequestAndParseResponse( + ctx, req, &body, + ) if err != nil { return nil, err } - if response.StatusCode != 200 { - var errorOutput []byte - if errorOutput, err = ioutil.ReadAll(response.Body); err != nil { - return nil, err - } - return nil, fmt.Errorf("HTTP %d : %s", response.StatusCode, errorOutput) - } - - var body struct { - ServerKeyList []ServerKeys `json:"server_keys"` - } - if err = json.NewDecoder(response.Body).Decode(&body); err != nil { - return nil, err - } - - result := map[PublicKeyRequest]ServerKeys{} - for _, keys := range body.ServerKeyList { - keys.FromServer = matrixServer - // TODO: What happens if the same key ID appears in multiple responses? - // We should probably take the response with the highest valid_until_ts. - for keyID := range keys.VerifyKeys { - result[PublicKeyRequest{keys.ServerName, keyID}] = keys - } - for keyID := range keys.OldVerifyKeys { - result[PublicKeyRequest{keys.ServerName, keyID}] = keys - } - } - return result, nil + return body.ServerKeyList, nil } // CreateMediaDownloadRequest creates a request for media on a homeserver and returns the http.Response or an error @@ -257,10 +245,70 @@ func (fc *Client) CreateMediaDownloadRequest( return nil, err } - return fc.doHTTPRequest(ctx, req) + return fc.DoHTTPRequest(ctx, req) } -func (fc *Client) doHTTPRequest(ctx context.Context, req *http.Request) (*http.Response, error) { +// DoRequestAndParseResponse calls DoHTTPRequest and then decodes the response. +// +// If the HTTP response is not a 200, an attempt is made to parse the response +// body into a gomatrix.RespError. In any case, a non-200 response will result +// in a gomatrix.HTTPError. +// +func (fc *Client) DoRequestAndParseResponse( + ctx context.Context, + req *http.Request, + result interface{}, +) error { + response, err := fc.DoHTTPRequest(ctx, req) + if response != nil { + defer response.Body.Close() // nolint: errcheck + } + if err != nil { + return err + } + + if response.StatusCode/100 != 2 { // not 2xx + // Adapted from https://github.com/matrix-org/gomatrix/blob/master/client.go + var contents []byte + contents, err = ioutil.ReadAll(response.Body) + if err != nil { + return err + } + + var wrap error + var respErr gomatrix.RespError + if _ = json.Unmarshal(contents, &respErr); respErr.ErrCode != "" { + wrap = respErr + } + + // If we failed to decode as RespError, don't just drop the HTTP body, include it in the + // HTTP error instead (e.g proxy errors which return HTML). + msg := "Failed to " + req.Method + " JSON to " + req.RequestURI + if wrap == nil { + msg = msg + ": " + string(contents) + } + + return gomatrix.HTTPError{ + Code: response.StatusCode, + Message: msg, + WrappedError: wrap, + } + } + + if err = json.NewDecoder(response.Body).Decode(result); err != nil { + return err + } + + return nil +} + +// DoHTTPRequest creates an outgoing request ID and adds it to the context +// before sending off the request and awaiting a response. +// +// If the returned error is nil, the Response will contain a non-nil +// Body which the caller is expected to close. +// +func (fc *Client) DoHTTPRequest(ctx context.Context, req *http.Request) (*http.Response, error) { reqID := util.RandomString(12) logger := util.GetLogger(ctx).WithField("server", req.URL.Host).WithField("out.req.ID", reqID) newCtx := util.ContextWithLogger(ctx, logger) diff --git a/vendor/src/github.com/matrix-org/gomatrixserverlib/federationclient.go b/vendor/src/github.com/matrix-org/gomatrixserverlib/federationclient.go index 06db3d861..299f08410 100644 --- a/vendor/src/github.com/matrix-org/gomatrixserverlib/federationclient.go +++ b/vendor/src/github.com/matrix-org/gomatrixserverlib/federationclient.go @@ -2,17 +2,13 @@ package gomatrixserverlib import ( "context" - "encoding/json" - "io/ioutil" "net/http" "net/url" - "github.com/matrix-org/gomatrix" - "github.com/matrix-org/util" "golang.org/x/crypto/ed25519" ) -// An FederationClient is a matrix federation client that adds +// A FederationClient is a matrix federation client that adds // "Authorization: X-Matrix" headers to requests that need ed25519 signatures type FederationClient struct { Client @@ -34,10 +30,6 @@ func NewFederationClient( } func (ac *FederationClient) doRequest(ctx context.Context, r FederationRequest, resBody interface{}) error { - reqID := util.RandomString(12) - logger := util.GetLogger(ctx).WithField("server", r.fields.Destination).WithField("out.req.ID", reqID) - newCtx := util.ContextWithLogger(ctx, logger) - if err := r.Sign(ac.serverName, ac.serverKeyID, ac.serverPrivateKey); err != nil { return err } @@ -47,52 +39,7 @@ func (ac *FederationClient) doRequest(ctx context.Context, r FederationRequest, return err } - logger.Infof("Outgoing request %s %s", req.Method, req.URL) - res, err := ac.client.Do(req.WithContext(newCtx)) - if res != nil { - defer res.Body.Close() // nolint: errcheck - } - - if err != nil { - logger.Infof("Outgoing request %s %s failed with %v", req.Method, req.URL, err) - return err - } - - contents, err := ioutil.ReadAll(res.Body) - - logger.Infof("Response %d from %s %s", res.StatusCode, req.Method, req.URL) - - if res.StatusCode/100 != 2 { // not 2xx - // Adapted from https://github.com/matrix-org/gomatrix/blob/master/client.go - var wrap error - var respErr gomatrix.RespError - if _ = json.Unmarshal(contents, &respErr); respErr.ErrCode != "" { - wrap = respErr - } - - // If we failed to decode as RespError, don't just drop the HTTP body, include it in the - // HTTP error instead (e.g proxy errors which return HTML). - msg := "Failed to " + r.Method() + " JSON to " + r.RequestURI() - if wrap == nil { - msg = msg + ": " + string(contents) - } - - return gomatrix.HTTPError{ - Code: res.StatusCode, - Message: msg, - WrappedError: wrap, - } - } - - if err != nil { - return err - } - - if resBody == nil { - return nil - } - - return json.Unmarshal(contents, resBody) + return ac.Client.DoRequestAndParseResponse(ctx, req, resBody) } var federationPathPrefix = "/_matrix/federation/v1" diff --git a/vendor/src/github.com/matrix-org/gomatrixserverlib/keyring.go b/vendor/src/github.com/matrix-org/gomatrixserverlib/keyring.go index 324e59d13..cd74de67f 100644 --- a/vendor/src/github.com/matrix-org/gomatrixserverlib/keyring.go +++ b/vendor/src/github.com/matrix-org/gomatrixserverlib/keyring.go @@ -17,6 +17,38 @@ type PublicKeyRequest struct { KeyID KeyID } +// PublicKeyNotExpired is a magic value for PublicKeyLookupResult.ExpiredTS: +// it indicates that this is an active key which has not yet expired +const PublicKeyNotExpired = Timestamp(0) + +// PublicKeyNotValid is a magic value for PublicKeyLookupResult.ValidUntilTS: +// it is used when we don't have a validity period for this key. Most likely +// it is an old key with an expiry date. +const PublicKeyNotValid = Timestamp(0) + +// A PublicKeyLookupResult is the result of looking up a server signing key. +type PublicKeyLookupResult struct { + VerifyKey + // if this key has expired, the time it stopped being valid for event signing in milliseconds. + // if the key has not expired, the magic value PublicKeyNotExpired. + ExpiredTS Timestamp + // When this result is valid until in milliseconds. + // if the key has expired, the magic value PublicKeyNotValid. + ValidUntilTS Timestamp +} + +// WasValidAt checks if this signing key is valid for an event signed at the +// given timestamp. +func (r PublicKeyLookupResult) WasValidAt(atTs Timestamp) bool { + if r.ExpiredTS != PublicKeyNotExpired && atTs >= r.ExpiredTS { + return false + } + if r.ValidUntilTS == PublicKeyNotValid || atTs > r.ValidUntilTS { + return false + } + return true +} + // A KeyFetcher is a way of fetching public keys in bulk. type KeyFetcher interface { // Lookup a batch of public keys. @@ -27,7 +59,7 @@ type KeyFetcher interface { // The result may have fewer (server name, key ID) pairs than were in the request. // The result may have more (server name, key ID) pairs than were in the request. // Returns an error if there was a problem fetching the keys. - FetchKeys(ctx context.Context, requests map[PublicKeyRequest]Timestamp) (map[PublicKeyRequest]ServerKeys, error) + FetchKeys(ctx context.Context, requests map[PublicKeyRequest]Timestamp) (map[PublicKeyRequest]PublicKeyLookupResult, error) } // A KeyDatabase is a store for caching public keys. @@ -40,7 +72,7 @@ type KeyDatabase interface { // to a concurrent FetchKeys(). This is acceptable since the database is // only used as a cache for the keys, so if a FetchKeys() races with a // StoreKeys() and some of the keys are missing they will be just be refetched. - StoreKeys(ctx context.Context, results map[PublicKeyRequest]ServerKeys) error + StoreKeys(ctx context.Context, results map[PublicKeyRequest]PublicKeyLookupResult) error } // A KeyRing stores keys for matrix servers and provides methods for verifying JSON messages. @@ -123,6 +155,8 @@ func (k KeyRing) VerifyJSONs(ctx context.Context, requests []VerifyJSONRequest) k.checkUsingKeys(requests, results, keyIDs, keysFromDatabase) for i := range k.KeyFetchers { + // TODO: we should distinguish here between expired keys, and those we don't have. + // If the key has expired, it's no use re-requesting it. keyRequests := k.publicKeyRequests(requests, results, keyIDs) if len(keyRequests) == 0 { // There aren't any keys to fetch so we can stop here. @@ -178,7 +212,7 @@ func (k *KeyRing) publicKeyRequests( func (k *KeyRing) checkUsingKeys( requests []VerifyJSONRequest, results []VerifyJSONResult, keyIDs [][]KeyID, - keys map[PublicKeyRequest]ServerKeys, + keys map[PublicKeyRequest]PublicKeyLookupResult, ) { for i := range requests { if results[i].Error == nil { @@ -187,13 +221,12 @@ func (k *KeyRing) checkUsingKeys( continue } for _, keyID := range keyIDs[i] { - serverKeys, ok := keys[PublicKeyRequest{requests[i].ServerName, keyID}] + serverKey, ok := keys[PublicKeyRequest{requests[i].ServerName, keyID}] if !ok { // No key for this key ID so we continue onto the next key ID. continue } - publicKey := serverKeys.PublicKey(keyID, requests[i].AtTS) - if publicKey == nil { + if !serverKey.WasValidAt(requests[i].AtTS) { // The key wasn't valid at the timestamp we needed it to be valid at. // So skip onto the next key. results[i].Error = fmt.Errorf( @@ -203,7 +236,7 @@ func (k *KeyRing) checkUsingKeys( continue } if err := VerifyJSON( - string(requests[i].ServerName), keyID, ed25519.PublicKey(publicKey), requests[i].Message, + string(requests[i].ServerName), keyID, ed25519.PublicKey(serverKey.Key), requests[i].Message, ); err != nil { // The signature wasn't valid, record the error and try the next key ID. results[i].Error = err @@ -229,13 +262,15 @@ type PerspectiveKeyFetcher struct { // FetchKeys implements KeyFetcher func (p *PerspectiveKeyFetcher) FetchKeys( ctx context.Context, requests map[PublicKeyRequest]Timestamp, -) (map[PublicKeyRequest]ServerKeys, error) { - results, err := p.Client.LookupServerKeys(ctx, p.PerspectiveServerName, requests) +) (map[PublicKeyRequest]PublicKeyLookupResult, error) { + serverKeys, err := p.Client.LookupServerKeys(ctx, p.PerspectiveServerName, requests) if err != nil { return nil, err } - for req, keys := range results { + results := map[PublicKeyRequest]PublicKeyLookupResult{} + + for _, keys := range serverKeys { var valid bool keyIDs, err := ListKeyIDs(string(p.PerspectiveServerName), keys.Raw) if err != nil { @@ -261,12 +296,16 @@ func (p *PerspectiveKeyFetcher) FetchKeys( return nil, fmt.Errorf("gomatrixserverlib: not signed with a known key for the perspective server") } - // Check that the keys are valid for the server. - checks, _, _ := CheckKeys(req.ServerName, time.Unix(0, 0), keys, nil) + // Check that the keys are valid for the server they claim to be + checks, _, _ := CheckKeys(keys.ServerName, time.Unix(0, 0), keys, nil) if !checks.AllChecksOK { // This is bad because it means that the perspective server was trying to feed us an invalid response. return nil, fmt.Errorf("gomatrixserverlib: key response from perspective server failed checks") } + + // TODO: What happens if the same key ID appears in multiple responses? + // We should probably take the response with the highest valid_until_ts. + mapServerKeysToPublicKeyLookupResult(keys, results) } return results, nil @@ -282,7 +321,7 @@ type DirectKeyFetcher struct { // FetchKeys implements KeyFetcher func (d *DirectKeyFetcher) FetchKeys( ctx context.Context, requests map[PublicKeyRequest]Timestamp, -) (map[PublicKeyRequest]ServerKeys, error) { +) (map[PublicKeyRequest]PublicKeyLookupResult, error) { byServer := map[ServerName]map[PublicKeyRequest]Timestamp{} for req, ts := range requests { server := byServer[req.ServerName] @@ -293,7 +332,7 @@ func (d *DirectKeyFetcher) FetchKeys( server[req] = ts } - results := map[PublicKeyRequest]ServerKeys{} + results := map[PublicKeyRequest]PublicKeyLookupResult{} for server, reqs := range byServer { // TODO: make these requests in parallel serverResults, err := d.fetchKeysForServer(ctx, server, reqs) @@ -310,19 +349,50 @@ func (d *DirectKeyFetcher) FetchKeys( func (d *DirectKeyFetcher) fetchKeysForServer( ctx context.Context, serverName ServerName, requests map[PublicKeyRequest]Timestamp, -) (map[PublicKeyRequest]ServerKeys, error) { - results, err := d.Client.LookupServerKeys(ctx, serverName, requests) +) (map[PublicKeyRequest]PublicKeyLookupResult, error) { + serverKeys, err := d.Client.LookupServerKeys(ctx, serverName, requests) if err != nil { return nil, err } - for req, keys := range results { + results := map[PublicKeyRequest]PublicKeyLookupResult{} + for _, keys := range serverKeys { // Check that the keys are valid for the server. - checks, _, _ := CheckKeys(req.ServerName, time.Unix(0, 0), keys, nil) + checks, _, _ := CheckKeys(serverName, time.Unix(0, 0), keys, nil) if !checks.AllChecksOK { return nil, fmt.Errorf("gomatrixserverlib: key response direct from %q failed checks", serverName) } + + // TODO: What happens if the same key ID appears in multiple responses? + // We should probably take the response with the highest valid_until_ts. + mapServerKeysToPublicKeyLookupResult(keys, results) } return results, nil } + +// mapServerKeysToPublicKeyLookupResult takes the (verified) result from a +// /key/v2/query call and inserts it into a PublicKeyRequest->PublicKeyLookupResult +// map. +func mapServerKeysToPublicKeyLookupResult(serverKeys ServerKeys, results map[PublicKeyRequest]PublicKeyLookupResult) { + for keyID, key := range serverKeys.VerifyKeys { + results[PublicKeyRequest{ + ServerName: serverKeys.ServerName, + KeyID: keyID, + }] = PublicKeyLookupResult{ + VerifyKey: key, + ValidUntilTS: serverKeys.ValidUntilTS, + ExpiredTS: PublicKeyNotExpired, + } + } + for keyID, key := range serverKeys.OldVerifyKeys { + results[PublicKeyRequest{ + ServerName: serverKeys.ServerName, + KeyID: keyID, + }] = PublicKeyLookupResult{ + VerifyKey: key.VerifyKey, + ValidUntilTS: PublicKeyNotValid, + ExpiredTS: key.ExpiredTS, + } + } +} diff --git a/vendor/src/github.com/matrix-org/gomatrixserverlib/keyring_test.go b/vendor/src/github.com/matrix-org/gomatrixserverlib/keyring_test.go index 5414e806e..07d1995ab 100644 --- a/vendor/src/github.com/matrix-org/gomatrixserverlib/keyring_test.go +++ b/vendor/src/github.com/matrix-org/gomatrixserverlib/keyring_test.go @@ -2,7 +2,6 @@ package gomatrixserverlib import ( "context" - "encoding/json" "testing" ) @@ -39,26 +38,44 @@ type testKeyDatabase struct{} func (db *testKeyDatabase) FetchKeys( ctx context.Context, requests map[PublicKeyRequest]Timestamp, -) (map[PublicKeyRequest]ServerKeys, error) { - results := map[PublicKeyRequest]ServerKeys{} - var keys ServerKeys - if err := json.Unmarshal([]byte(testKeys), &keys); err != nil { - return nil, err - } +) (map[PublicKeyRequest]PublicKeyLookupResult, error) { + results := map[PublicKeyRequest]PublicKeyLookupResult{} req1 := PublicKeyRequest{"localhost:8800", "ed25519:old"} req2 := PublicKeyRequest{"localhost:8800", "ed25519:a_Obwu"} for req := range requests { - if req == req1 || req == req2 { - results[req] = keys + if req == req1 { + vk := VerifyKey{} + err := vk.Key.Decode("O2onvM62pC1io6jQKm8Nc2UyFXcd4kOmOsBIoYtZ2ik") + if err != nil { + return nil, err + } + results[req] = PublicKeyLookupResult{ + VerifyKey: vk, + ValidUntilTS: PublicKeyNotValid, + ExpiredTS: 929059200, + } + } + + if req == req2 { + vk := VerifyKey{} + err := vk.Key.Decode("2UwTWD4+tgTgENV7znGGNqhAOGY+BW1mRAnC6W6FBQg") + if err != nil { + return nil, err + } + results[req] = PublicKeyLookupResult{ + VerifyKey: vk, + ValidUntilTS: 1493142432964, + ExpiredTS: PublicKeyNotExpired, + } } } return results, nil } func (db *testKeyDatabase) StoreKeys( - ctx context.Context, requests map[PublicKeyRequest]ServerKeys, + ctx context.Context, requests map[PublicKeyRequest]PublicKeyLookupResult, ) error { return nil } @@ -136,12 +153,12 @@ var testErrorStore = erroringKeyDatabaseError(2) func (e *erroringKeyDatabase) FetchKeys( ctx context.Context, requests map[PublicKeyRequest]Timestamp, -) (map[PublicKeyRequest]ServerKeys, error) { +) (map[PublicKeyRequest]PublicKeyLookupResult, error) { return nil, &testErrorFetch } func (e *erroringKeyDatabase) StoreKeys( - ctx context.Context, keys map[PublicKeyRequest]ServerKeys, + ctx context.Context, keys map[PublicKeyRequest]PublicKeyLookupResult, ) error { return &testErrorStore } diff --git a/vendor/src/github.com/matrix-org/gomatrixserverlib/keys.go b/vendor/src/github.com/matrix-org/gomatrixserverlib/keys.go index 099f34046..98ff87fd6 100644 --- a/vendor/src/github.com/matrix-org/gomatrixserverlib/keys.go +++ b/vendor/src/github.com/matrix-org/gomatrixserverlib/keys.go @@ -37,8 +37,6 @@ type ServerName string type ServerKeys struct { // Copy of the raw JSON for signature checking. Raw []byte - // The server the raw JSON was downloaded from. - FromServer ServerName // The decoded JSON fields. ServerKeyFields } @@ -140,7 +138,6 @@ func FetchKeysDirect(serverName ServerName, addr, sni string) (*ServerKeys, *tls return nil, nil, err } var keys ServerKeys - keys.FromServer = serverName if err = json.NewDecoder(response.Body).Decode(&keys); err != nil { return nil, nil, err } diff --git a/vendor/src/github.com/matrix-org/gomatrixserverlib/linter.json b/vendor/src/github.com/matrix-org/gomatrixserverlib/linter.json index 8ad86b870..aae749546 100644 --- a/vendor/src/github.com/matrix-org/gomatrixserverlib/linter.json +++ b/vendor/src/github.com/matrix-org/gomatrixserverlib/linter.json @@ -9,7 +9,7 @@ "golint", "varcheck", "structcheck", - "aligncheck", + "maligned", "ineffassign", "gas", "misspell", From 561315e1d616fd490c1ec720e7a825e3ed916249 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Nov 2017 10:12:02 +0000 Subject: [PATCH 23/29] Add gofmt linter (#339) * Check that gofmt has been run * gofmt files * Tabs and spaces --- linter-fast.json | 1 + linter.json | 1 + .../dendrite/clientapi/routing/createroom.go | 2 +- .../matrix-org/dendrite/clientapi/routing/voip.go | 12 ++++++------ .../dendrite/cmd/dendrite-room-server/main.go | 2 +- .../matrix-org/dendrite/common/config/config.go | 2 +- src/github.com/matrix-org/dendrite/common/log.go | 2 +- .../federationsender/consumers/roomserver.go | 2 +- .../federationsender/queue/destinationqueue.go | 2 +- .../dendrite/federationsender/queue/queue.go | 2 +- .../dendrite/mediaapi/fileutils/fileutils.go | 2 +- .../matrix-org/dendrite/mediaapi/routing/download.go | 2 +- .../dendrite/mediaapi/thumbnailer/thumbnailer.go | 2 +- .../mediaapi/thumbnailer/thumbnailer_bimg.go | 2 +- .../mediaapi/thumbnailer/thumbnailer_nfnt.go | 2 +- .../dendrite/publicroomsapi/consumers/roomserver.go | 2 +- .../dendrite/syncapi/consumers/clientapi.go | 2 +- .../dendrite/syncapi/consumers/roomserver.go | 2 +- .../syncapi/storage/output_room_events_table.go | 2 +- .../matrix-org/dendrite/syncapi/sync/request.go | 2 +- 20 files changed, 25 insertions(+), 23 deletions(-) diff --git a/linter-fast.json b/linter-fast.json index aa86054e8..81188d79c 100644 --- a/linter-fast.json +++ b/linter-fast.json @@ -11,6 +11,7 @@ "misspell", "errcheck", "vet", + "gofmt", "goconst" ] } diff --git a/linter.json b/linter.json index 511c244b6..de4f2bf6a 100644 --- a/linter.json +++ b/linter.json @@ -17,6 +17,7 @@ "errcheck", "vet", "megacheck", + "gofmt", "goconst" ] } diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/createroom.go b/src/github.com/matrix-org/dendrite/clientapi/routing/createroom.go index 078a7319b..c84c1f7cf 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/createroom.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/createroom.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" @@ -33,6 +32,7 @@ import ( "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" ) // https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-createroom diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go b/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go index e699a91f2..c4c48bd73 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go @@ -24,16 +24,16 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/util" - "github.com/matrix-org/dendrite/clientapi/httputil" ) type turnServerResponse struct { - Username string `json:"username"` - Password string `json:"password"` - URIs []string `json:"uris"` - TTL int `json:"ttl"` + Username string `json:"username"` + Password string `json:"password"` + URIs []string `json:"uris"` + TTL int `json:"ttl"` } // RequestTurnServer implements: @@ -54,7 +54,7 @@ func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg config.D resp := turnServerResponse{ URIs: turnConfig.URIs, - TTL: int(duration.Seconds()), + TTL: int(duration.Seconds()), } if turnConfig.SharedSecret != "" { diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go index f607d1ec2..06773972a 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go @@ -20,7 +20,6 @@ import ( _ "net/http/pprof" "os" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/alias" @@ -28,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/query" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" sarama "gopkg.in/Shopify/sarama.v1" ) diff --git a/src/github.com/matrix-org/dendrite/common/config/config.go b/src/github.com/matrix-org/dendrite/common/config/config.go index 82bdc3dca..65a9f8986 100644 --- a/src/github.com/matrix-org/dendrite/common/config/config.go +++ b/src/github.com/matrix-org/dendrite/common/config/config.go @@ -25,8 +25,8 @@ import ( "strings" "time" - "github.com/sirupsen/logrus" "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" "golang.org/x/crypto/ed25519" "gopkg.in/yaml.v2" diff --git a/src/github.com/matrix-org/dendrite/common/log.go b/src/github.com/matrix-org/dendrite/common/log.go index f5c5bf340..fbfa34783 100644 --- a/src/github.com/matrix-org/dendrite/common/log.go +++ b/src/github.com/matrix-org/dendrite/common/log.go @@ -18,8 +18,8 @@ import ( "os" "path/filepath" - "github.com/sirupsen/logrus" "github.com/matrix-org/dugong" + "github.com/sirupsen/logrus" ) type utcFormatter struct { diff --git a/src/github.com/matrix-org/dendrite/federationsender/consumers/roomserver.go b/src/github.com/matrix-org/dendrite/federationsender/consumers/roomserver.go index a396aaf6d..45e48f166 100644 --- a/src/github.com/matrix-org/dendrite/federationsender/consumers/roomserver.go +++ b/src/github.com/matrix-org/dendrite/federationsender/consumers/roomserver.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/federationsender/queue" @@ -27,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" + log "github.com/sirupsen/logrus" sarama "gopkg.in/Shopify/sarama.v1" ) diff --git a/src/github.com/matrix-org/dendrite/federationsender/queue/destinationqueue.go b/src/github.com/matrix-org/dendrite/federationsender/queue/destinationqueue.go index 265dbbc3b..2013a7a4b 100644 --- a/src/github.com/matrix-org/dendrite/federationsender/queue/destinationqueue.go +++ b/src/github.com/matrix-org/dendrite/federationsender/queue/destinationqueue.go @@ -20,8 +20,8 @@ import ( "sync" "time" - log "github.com/sirupsen/logrus" "github.com/matrix-org/gomatrixserverlib" + log "github.com/sirupsen/logrus" ) // destinationQueue is a queue of events for a single destination. diff --git a/src/github.com/matrix-org/dendrite/federationsender/queue/queue.go b/src/github.com/matrix-org/dendrite/federationsender/queue/queue.go index 87b040be4..d31c12f99 100644 --- a/src/github.com/matrix-org/dendrite/federationsender/queue/queue.go +++ b/src/github.com/matrix-org/dendrite/federationsender/queue/queue.go @@ -18,8 +18,8 @@ import ( "fmt" "sync" - log "github.com/sirupsen/logrus" "github.com/matrix-org/gomatrixserverlib" + log "github.com/sirupsen/logrus" ) // OutgoingQueues is a collection of queues for sending transactions to other diff --git a/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go b/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go index 1d5275287..36b2c5b89 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go @@ -25,9 +25,9 @@ import ( "path/filepath" "strings" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/types" + log "github.com/sirupsen/logrus" ) // GetPathFromBase64Hash evaluates the path to a media file from its Base64Hash diff --git a/src/github.com/matrix-org/dendrite/mediaapi/routing/download.go b/src/github.com/matrix-org/dendrite/mediaapi/routing/download.go index eba3038fd..2ce2993f8 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/routing/download.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/routing/download.go @@ -28,7 +28,6 @@ import ( "strings" "sync" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/fileutils" @@ -38,6 +37,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) const mediaIDCharacters = "A-Za-z0-9_=-" diff --git a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go index efef03b5f..61b66ebc6 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go @@ -22,10 +22,10 @@ import ( "path/filepath" "sync" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" + log "github.com/sirupsen/logrus" ) type thumbnailFitness struct { diff --git a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go index 178028684..db6f23ace 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go @@ -21,10 +21,10 @@ import ( "os" "time" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" + log "github.com/sirupsen/logrus" "gopkg.in/h2non/bimg.v1" ) diff --git a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go index 4348e2bcb..e92767317 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go @@ -28,11 +28,11 @@ import ( "os" "time" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/nfnt/resize" + log "github.com/sirupsen/logrus" ) // GenerateThumbnails generates the configured thumbnail sizes for the source file diff --git a/src/github.com/matrix-org/dendrite/publicroomsapi/consumers/roomserver.go b/src/github.com/matrix-org/dendrite/publicroomsapi/consumers/roomserver.go index 46e38b10a..b7d42b111 100644 --- a/src/github.com/matrix-org/dendrite/publicroomsapi/consumers/roomserver.go +++ b/src/github.com/matrix-org/dendrite/publicroomsapi/consumers/roomserver.go @@ -18,11 +18,11 @@ import ( "context" "encoding/json" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver/api" + log "github.com/sirupsen/logrus" sarama "gopkg.in/Shopify/sarama.v1" ) diff --git a/src/github.com/matrix-org/dendrite/syncapi/consumers/clientapi.go b/src/github.com/matrix-org/dendrite/syncapi/consumers/clientapi.go index 4eb4be39f..d05a76920 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/consumers/clientapi.go +++ b/src/github.com/matrix-org/dendrite/syncapi/consumers/clientapi.go @@ -18,11 +18,11 @@ import ( "context" "encoding/json" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" + log "github.com/sirupsen/logrus" sarama "gopkg.in/Shopify/sarama.v1" ) diff --git a/src/github.com/matrix-org/dendrite/syncapi/consumers/roomserver.go b/src/github.com/matrix-org/dendrite/syncapi/consumers/roomserver.go index cefd3f1e2..677eeb42b 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/consumers/roomserver.go +++ b/src/github.com/matrix-org/dendrite/syncapi/consumers/roomserver.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" @@ -27,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" + log "github.com/sirupsen/logrus" sarama "gopkg.in/Shopify/sarama.v1" ) diff --git a/src/github.com/matrix-org/dendrite/syncapi/storage/output_room_events_table.go b/src/github.com/matrix-org/dendrite/syncapi/storage/output_room_events_table.go index 8f268b484..fb00ad842 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/storage/output_room_events_table.go +++ b/src/github.com/matrix-org/dendrite/syncapi/storage/output_room_events_table.go @@ -18,11 +18,11 @@ import ( "context" "database/sql" - log "github.com/sirupsen/logrus" "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" + log "github.com/sirupsen/logrus" ) const outputRoomEventsSchema = ` diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/request.go b/src/github.com/matrix-org/dendrite/syncapi/sync/request.go index 4754e6649..7dec55fe1 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/request.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/request.go @@ -20,9 +20,9 @@ import ( "strconv" "time" - log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" ) const defaultSyncTimeout = time.Duration(30) * time.Second From 4b280943dceb3ad41c16bf4fde9b58ec9c378e1c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Nov 2017 15:28:15 +0000 Subject: [PATCH 24/29] Fix SQL --- .../dendrite/clientapi/auth/storage/devices/devices_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go index 903471afe..c8ece6b91 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go @@ -54,7 +54,7 @@ const insertDeviceSQL = "" + "INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5)" const selectDeviceByTokenSQL = "" + - "SELECT device_id, localpart, display_name FROM device_devices WHERE access_token = $1" + "SELECT device_id, localpart FROM device_devices WHERE access_token = $1" const selectDeviceByIDSQL = "" + "SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2" From bdc44c4bde71b9010ef531366a07ecccce7b98a6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Nov 2017 15:46:52 +0000 Subject: [PATCH 25/29] Log errors when there is an error validating token (#340) --- src/github.com/matrix-org/dendrite/clientapi/auth/auth.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/auth.go b/src/github.com/matrix-org/dendrite/clientapi/auth/auth.go index b6a3efc28..56ee31ecb 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/auth.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/auth.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/util" ) @@ -62,10 +63,8 @@ func VerifyAccessToken(req *http.Request, deviceDB DeviceDatabase) (device *auth JSON: jsonerror.UnknownToken("Unknown token"), } } else { - resErr = &util.JSONResponse{ - Code: 500, - JSON: jsonerror.Unknown("Failed to check access token"), - } + jsonErr := httputil.LogThenError(req, err) + resErr = &jsonErr } } return From 8599a36fa681e966abedd31ecac7fe98caa3fed1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Nov 2017 17:35:28 +0000 Subject: [PATCH 26/29] Use a Postgres database rather than Memory for Naffka (#337) * Update naffka dep * User Postgres database rather than Memory for Naffka --- .../cmd/dendrite-monolith-server/main.go | 17 +- .../dendrite/common/config/config.go | 4 + vendor/manifest | 2 +- .../matrix-org/naffka/memorydatabase.go | 14 +- .../github.com/matrix-org/naffka/naffka.go | 188 ++++++----- .../matrix-org/naffka/naffka_test.go | 140 +++++++++ .../matrix-org/naffka/postgresqldatabase.go | 296 ++++++++++++++++++ 7 files changed, 567 insertions(+), 94 deletions(-) create mode 100644 vendor/src/github.com/matrix-org/naffka/postgresqldatabase.go diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go index ea71fed98..fafd4cb82 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go @@ -16,6 +16,7 @@ package main import ( "context" + "database/sql" "flag" "net/http" "os" @@ -199,7 +200,21 @@ func (m *monolith) setupFederation() { func (m *monolith) setupKafka() { if m.cfg.Kafka.UseNaffka { - naff, err := naffka.New(&naffka.MemoryDatabase{}) + db, err := sql.Open("postgres", string(m.cfg.Database.Naffka)) + if err != nil { + log.WithFields(log.Fields{ + log.ErrorKey: err, + }).Panic("Failed to open naffka database") + } + + naffkaDB, err := naffka.NewPostgresqlDatabase(db) + if err != nil { + log.WithFields(log.Fields{ + log.ErrorKey: err, + }).Panic("Failed to setup naffka database") + } + + naff, err := naffka.New(naffkaDB) if err != nil { log.WithFields(log.Fields{ log.ErrorKey: err, diff --git a/src/github.com/matrix-org/dendrite/common/config/config.go b/src/github.com/matrix-org/dendrite/common/config/config.go index 65a9f8986..797889028 100644 --- a/src/github.com/matrix-org/dendrite/common/config/config.go +++ b/src/github.com/matrix-org/dendrite/common/config/config.go @@ -148,6 +148,8 @@ type Dendrite struct { // The PublicRoomsAPI database stores information used to compute the public // room directory. It is only accessed by the PublicRoomsAPI server. PublicRoomsAPI DataSource `yaml:"public_rooms_api"` + // The Naffka database is used internally by the naffka library, if used. + Naffka DataSource `yaml:"naffka,omitempty"` } `yaml:"database"` // TURN Server Config @@ -386,6 +388,8 @@ func (config *Dendrite) check(monolithic bool) error { if !monolithic { problems = append(problems, fmt.Sprintf("naffka can only be used in a monolithic server")) } + + checkNotEmpty("database.naffka", string(config.Database.Naffka)) } else { // If we aren't using naffka then we need to have at least one kafka // server to talk to. diff --git a/vendor/manifest b/vendor/manifest index ff61798d5..ba6dab4c4 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -141,7 +141,7 @@ { "importpath": "github.com/matrix-org/naffka", "repository": "https://github.com/matrix-org/naffka", - "revision": "d28656e34f96a8eeaab53e3b7678c9ce14af5786", + "revision": "662bfd0841d0194bfe0a700d54226bb96eac574d", "branch": "master" }, { diff --git a/vendor/src/github.com/matrix-org/naffka/memorydatabase.go b/vendor/src/github.com/matrix-org/naffka/memorydatabase.go index 05d1f3ee6..6166c4934 100644 --- a/vendor/src/github.com/matrix-org/naffka/memorydatabase.go +++ b/vendor/src/github.com/matrix-org/naffka/memorydatabase.go @@ -8,7 +8,8 @@ import ( // A MemoryDatabase stores the message history as arrays in memory. // It can be used to run unit tests. // If the process is stopped then any messages that haven't been -// processed by a consumer are lost forever. +// processed by a consumer are lost forever and all offsets become +// invalid. type MemoryDatabase struct { topicsMutex sync.Mutex topics map[string]*memoryDatabaseTopic @@ -58,10 +59,7 @@ func (m *MemoryDatabase) getTopic(topicName string) *memoryDatabaseTopic { // StoreMessages implements Database func (m *MemoryDatabase) StoreMessages(topic string, messages []Message) error { - if err := m.getTopic(topic).addMessages(messages); err != nil { - return err - } - return nil + return m.getTopic(topic).addMessages(messages) } // FetchMessages implements Database @@ -73,10 +71,10 @@ func (m *MemoryDatabase) FetchMessages(topic string, startOffset, endOffset int6 if startOffset >= endOffset { return nil, fmt.Errorf("start offset %d greater than or equal to end offset %d", startOffset, endOffset) } - if startOffset < -1 { - return nil, fmt.Errorf("start offset %d less than -1", startOffset) + if startOffset < 0 { + return nil, fmt.Errorf("start offset %d less than 0", startOffset) } - return messages[startOffset+1 : endOffset], nil + return messages[startOffset:endOffset], nil } // MaxOffsets implements Database diff --git a/vendor/src/github.com/matrix-org/naffka/naffka.go b/vendor/src/github.com/matrix-org/naffka/naffka.go index d429ffdaa..e384b04ff 100644 --- a/vendor/src/github.com/matrix-org/naffka/naffka.go +++ b/vendor/src/github.com/matrix-org/naffka/naffka.go @@ -13,6 +13,7 @@ import ( // single go process. It implements both the sarama.SyncProducer and the // sarama.Consumer interfaces. This means it can act as a drop in replacement // for kafka for testing or single instance deployment. +// Does not support multiple partitions. type Naffka struct { db Database topicsMutex sync.Mutex @@ -28,6 +29,7 @@ func New(db Database) (*Naffka, error) { } for topicName, offset := range maxOffsets { n.topics[topicName] = &topic{ + db: db, topicName: topicName, nextOffset: offset + 1, } @@ -64,7 +66,7 @@ type Database interface { // So for a given topic the message with offset n+1 is stored after the // the message with offset n. StoreMessages(topic string, messages []Message) error - // FetchMessages fetches all messages with an offset greater than but not + // FetchMessages fetches all messages with an offset greater than and // including startOffset and less than but not including endOffset. // The range of offsets requested must not overlap with those stored by a // concurrent StoreMessages. The message offsets within the requested range @@ -138,6 +140,7 @@ func (n *Naffka) Partitions(topic string) ([]int32, error) { } // ConsumePartition implements sarama.Consumer +// Note: offset is *inclusive*, i.e. it will include the message with that offset. func (n *Naffka) ConsumePartition(topic string, partition int32, offset int64) (sarama.PartitionConsumer, error) { if partition != 0 { return nil, fmt.Errorf("Unknown partition ID %d", partition) @@ -166,13 +169,16 @@ func (n *Naffka) Close() error { const channelSize = 1024 +// partitionConsumer ensures that all messages written to a particular +// topic, from an offset, get sent in order to a channel. +// Implements sarama.PartitionConsumer type partitionConsumer struct { topic *topic messages chan *sarama.ConsumerMessage - // Whether the consumer is ready for new messages or whether it - // is catching up on historic messages. + // Whether the consumer is in "catchup" mode or not. + // See "catchup" function for details. // Reads and writes to this field are proctected by the topic mutex. - ready bool + catchingUp bool } // AsyncClose implements sarama.PartitionConsumer @@ -201,66 +207,101 @@ func (c *partitionConsumer) HighWaterMarkOffset() int64 { return c.topic.highwaterMark() } -// block writes the message to the consumer blocking until the consumer is ready -// to add the message to the channel. Once the message is successfully added to -// the channel it will catch up by pulling historic messsages from the database. -func (c *partitionConsumer) block(cmsg *sarama.ConsumerMessage) { - c.messages <- cmsg - c.catchup(cmsg.Offset) +// catchup makes the consumer go into "catchup" mode, where messages are read +// from the database instead of directly from producers. +// Once the consumer is up to date, i.e. no new messages in the database, then +// the consumer will go back into normal mode where new messages are written +// directly to the channel. +// Must be called with the c.topic.mutex lock +func (c *partitionConsumer) catchup(fromOffset int64) { + // If we're already in catchup mode or up to date, noop + if c.catchingUp || fromOffset == c.topic.nextOffset { + return + } + + c.catchingUp = true + + // Due to the checks above there can only be one of these goroutines + // running at a time + go func() { + for { + // Check if we're up to date yet. If we are we exit catchup mode. + c.topic.mutex.Lock() + nextOffset := c.topic.nextOffset + if fromOffset == nextOffset { + c.catchingUp = false + c.topic.mutex.Unlock() + return + } + c.topic.mutex.Unlock() + + // Limit the number of messages we request from the database to be the + // capacity of the channel. + if nextOffset > fromOffset+int64(cap(c.messages)) { + nextOffset = fromOffset + int64(cap(c.messages)) + } + // Fetch the messages from the database. + msgs, err := c.topic.db.FetchMessages(c.topic.topicName, fromOffset, nextOffset) + if err != nil { + // TODO: Add option to write consumer errors to an errors channel + // as an alternative to logging the errors. + log.Print("Error reading messages: ", err) + // Wait before retrying. + // TODO: Maybe use an exponentional backoff scheme here. + // TODO: This timeout should take account of all the other goroutines + // that might be doing the same thing. (If there are a 10000 consumers + // then we don't want to end up retrying every millisecond) + time.Sleep(10 * time.Second) + continue + } + if len(msgs) == 0 { + // This should only happen if the database is corrupted and has lost the + // messages between the requested offsets. + log.Fatalf("Corrupt database returned no messages between %d and %d", fromOffset, nextOffset) + } + + // Pass the messages into the consumer channel. + // Blocking each write until the channel has enough space for the message. + for i := range msgs { + c.messages <- msgs[i].consumerMessage(c.topic.topicName) + } + // Update our the offset for the next loop iteration. + fromOffset = msgs[len(msgs)-1].Offset + 1 + } + }() } -// catchup reads historic messages from the database until the consumer has caught -// up on all the historic messages. -func (c *partitionConsumer) catchup(fromOffset int64) { - for { - // First check if we have caught up. - caughtUp, nextOffset := c.topic.hasCaughtUp(c, fromOffset) - if caughtUp { - return - } - // Limit the number of messages we request from the database to be the - // capacity of the channel. - if nextOffset > fromOffset+int64(cap(c.messages)) { - nextOffset = fromOffset + int64(cap(c.messages)) - } - // Fetch the messages from the database. - msgs, err := c.topic.db.FetchMessages(c.topic.topicName, fromOffset, nextOffset) - if err != nil { - // TODO: Add option to write consumer errors to an errors channel - // as an alternative to logging the errors. - log.Print("Error reading messages: ", err) - // Wait before retrying. - // TODO: Maybe use an exponentional backoff scheme here. - // TODO: This timeout should take account of all the other goroutines - // that might be doing the same thing. (If there are a 10000 consumers - // then we don't want to end up retrying every millisecond) - time.Sleep(10 * time.Second) - continue - } - if len(msgs) == 0 { - // This should only happen if the database is corrupted and has lost the - // messages between the requested offsets. - log.Fatalf("Corrupt database returned no messages between %d and %d", fromOffset, nextOffset) - } +// notifyNewMessage tells the consumer about a new message +// Must be called with the c.topic.mutex lock +func (c *partitionConsumer) notifyNewMessage(cmsg *sarama.ConsumerMessage) { + // If we're in "catchup" mode then the catchup routine will send the + // message later, since cmsg has already been written to the database + if c.catchingUp { + return + } - // Pass the messages into the consumer channel. - // Blocking each write until the channel has enough space for the message. - for i := range msgs { - c.messages <- msgs[i].consumerMessage(c.topic.topicName) - } - // Update our the offset for the next loop iteration. - fromOffset = msgs[len(msgs)-1].Offset + // Otherwise, lets try writing the message directly to the channel + select { + case c.messages <- cmsg: + default: + // The messages channel has filled up, so lets go into catchup + // mode. Once the channel starts being read from again messages + // will be read from the database + c.catchup(cmsg.Offset) } } type topic struct { - db Database - topicName string - mutex sync.Mutex - consumers []*partitionConsumer + db Database + topicName string + mutex sync.Mutex + consumers []*partitionConsumer + // nextOffset is the offset that will be assigned to the next message in + // this topic, i.e. one greater than the last message offset. nextOffset int64 } +// send writes messages to a topic. func (t *topic) send(now time.Time, pmsgs []*sarama.ProducerMessage) error { var err error // Encode the message keys and values. @@ -298,21 +339,10 @@ func (t *topic) send(now time.Time, pmsgs []*sarama.ProducerMessage) error { t.nextOffset = offset // Now notify the consumers about the messages. - for i := range msgs { - cmsg := msgs[i].consumerMessage(t.topicName) + for _, msg := range msgs { + cmsg := msg.consumerMessage(t.topicName) for _, c := range t.consumers { - if c.ready { - select { - case c.messages <- cmsg: - default: - // The consumer wasn't ready to receive a message because - // the channel buffer was full. - // Fork a goroutine to send the message so that we don't - // block sending messages to the other consumers. - c.ready = false - go c.block(cmsg) - } - } + c.notifyNewMessage(cmsg) } } @@ -330,27 +360,17 @@ func (t *topic) consume(offset int64) *partitionConsumer { offset = t.nextOffset } if offset == sarama.OffsetOldest { - offset = -1 + offset = 0 } c.messages = make(chan *sarama.ConsumerMessage, channelSize) t.consumers = append(t.consumers, c) - // Start catching up on historic messages in the background. - go c.catchup(offset) - return c -} -func (t *topic) hasCaughtUp(c *partitionConsumer, offset int64) (bool, int64) { - t.mutex.Lock() - defer t.mutex.Unlock() - // Check if we have caught up while holding a lock on the topic so there - // isn't a way for our check to race with a new message being sent on the topic. - if offset+1 == t.nextOffset { - // We've caught up, the consumer can now receive messages as they are - // sent rather than fetching them from the database. - c.ready = true - return true, t.nextOffset + // If we're not streaming from the latest offset we need to go into + // "catchup" mode + if offset != t.nextOffset { + c.catchup(offset) } - return false, t.nextOffset + return c } func (t *topic) highwaterMark() int64 { diff --git a/vendor/src/github.com/matrix-org/naffka/naffka_test.go b/vendor/src/github.com/matrix-org/naffka/naffka_test.go index d1a267102..dd69c95fc 100644 --- a/vendor/src/github.com/matrix-org/naffka/naffka_test.go +++ b/vendor/src/github.com/matrix-org/naffka/naffka_test.go @@ -1,6 +1,7 @@ package naffka import ( + "strconv" "testing" "time" @@ -84,3 +85,142 @@ func TestDelayedReceive(t *testing.T) { t.Fatalf("wrong value: wanted %q got %q", value, string(result.Value)) } } + +func TestCatchup(t *testing.T) { + naffka, err := New(&MemoryDatabase{}) + if err != nil { + t.Fatal(err) + } + producer := sarama.SyncProducer(naffka) + consumer := sarama.Consumer(naffka) + + const topic = "testTopic" + const value = "Hello, World" + + message := sarama.ProducerMessage{ + Value: sarama.StringEncoder(value), + Topic: topic, + } + + if _, _, err = producer.SendMessage(&message); err != nil { + t.Fatal(err) + } + + c, err := consumer.ConsumePartition(topic, 0, sarama.OffsetOldest) + if err != nil { + t.Fatal(err) + } + + var result *sarama.ConsumerMessage + select { + case result = <-c.Messages(): + case _ = <-time.NewTimer(10 * time.Second).C: + t.Fatal("expected to receive a message") + } + + if string(result.Value) != value { + t.Fatalf("wrong value: wanted %q got %q", value, string(result.Value)) + } + + currOffset := result.Offset + + const value2 = "Hello, World2" + const value3 = "Hello, World3" + + _, _, err = producer.SendMessage(&sarama.ProducerMessage{ + Value: sarama.StringEncoder(value2), + Topic: topic, + }) + if err != nil { + t.Fatal(err) + } + + _, _, err = producer.SendMessage(&sarama.ProducerMessage{ + Value: sarama.StringEncoder(value3), + Topic: topic, + }) + if err != nil { + t.Fatal(err) + } + + t.Logf("Streaming from %q", currOffset+1) + + c2, err := consumer.ConsumePartition(topic, 0, currOffset+1) + if err != nil { + t.Fatal(err) + } + + var result2 *sarama.ConsumerMessage + select { + case result2 = <-c2.Messages(): + case _ = <-time.NewTimer(10 * time.Second).C: + t.Fatal("expected to receive a message") + } + + if string(result2.Value) != value2 { + t.Fatalf("wrong value: wanted %q got %q", value2, string(result2.Value)) + } +} + +func TestChannelSaturation(t *testing.T) { + // The channel returned by c.Messages() has a fixed capacity + + naffka, err := New(&MemoryDatabase{}) + if err != nil { + t.Fatal(err) + } + producer := sarama.SyncProducer(naffka) + consumer := sarama.Consumer(naffka) + const topic = "testTopic" + const baseValue = "testValue: " + + c, err := consumer.ConsumePartition(topic, 0, sarama.OffsetOldest) + if err != nil { + t.Fatal(err) + } + + channelSize := cap(c.Messages()) + + // We want to send enough messages to fill up the channel, so lets double + // the size of the channel. And add three in case its a zero sized channel + numberMessagesToSend := 2*channelSize + 3 + + var sentMessages []string + + for i := 0; i < numberMessagesToSend; i++ { + value := baseValue + strconv.Itoa(i) + + message := sarama.ProducerMessage{ + Topic: topic, + Value: sarama.StringEncoder(value), + } + + sentMessages = append(sentMessages, value) + + if _, _, err = producer.SendMessage(&message); err != nil { + t.Fatal(err) + } + } + + var result *sarama.ConsumerMessage + + j := 0 + for ; j < numberMessagesToSend; j++ { + select { + case result = <-c.Messages(): + case _ = <-time.NewTimer(10 * time.Second).C: + t.Fatalf("failed to receive message %d out of %d", j+1, numberMessagesToSend) + } + + expectedValue := sentMessages[j] + if string(result.Value) != expectedValue { + t.Fatalf("wrong value: wanted %q got %q", expectedValue, string(result.Value)) + } + } + + select { + case result = <-c.Messages(): + t.Fatalf("expected to only receive %d messages", numberMessagesToSend) + default: + } +} diff --git a/vendor/src/github.com/matrix-org/naffka/postgresqldatabase.go b/vendor/src/github.com/matrix-org/naffka/postgresqldatabase.go new file mode 100644 index 000000000..d121630cc --- /dev/null +++ b/vendor/src/github.com/matrix-org/naffka/postgresqldatabase.go @@ -0,0 +1,296 @@ +package naffka + +import ( + "database/sql" + "sync" + "time" +) + +const postgresqlSchema = ` +-- The topic table assigns each topic a unique numeric ID. +CREATE SEQUENCE IF NOT EXISTS naffka_topic_nid_seq; +CREATE TABLE IF NOT EXISTS naffka_topics ( + topic_name TEXT PRIMARY KEY, + topic_nid BIGINT NOT NULL DEFAULT nextval('naffka_topic_nid_seq') +); + +-- The messages table contains the actual messages. +CREATE TABLE IF NOT EXISTS naffka_messages ( + topic_nid BIGINT NOT NULL, + message_offset BIGINT NOT NULL, + message_key BYTEA NOT NULL, + message_value BYTEA NOT NULL, + message_timestamp_ns BIGINT NOT NULL, + UNIQUE (topic_nid, message_offset) +); +` + +const insertTopicSQL = "" + + "INSERT INTO naffka_topics (topic_name) VALUES ($1)" + + " ON CONFLICT DO NOTHING" + + " RETURNING (topic_nid)" + +const selectTopicSQL = "" + + "SELECT topic_nid FROM naffka_topics WHERE topic_name = $1" + +const selectTopicsSQL = "" + + "SELECT topic_name, topic_nid FROM naffka_topics" + +const insertMessageSQL = "" + + "INSERT INTO naffka_messages (topic_nid, message_offset, message_key, message_value, message_timestamp_ns)" + + " VALUES ($1, $2, $3, $4, $5)" + +const selectMessagesSQL = "" + + "SELECT message_offset, message_key, message_value, message_timestamp_ns" + + " FROM naffka_messages WHERE topic_nid = $1 AND $2 <= message_offset AND message_offset < $3" + + " ORDER BY message_offset ASC" + +const selectMaxOffsetSQL = "" + + "SELECT message_offset FROM naffka_messages WHERE topic_nid = $1" + + " ORDER BY message_offset DESC LIMIT 1" + +type postgresqlDatabase struct { + db *sql.DB + topicsMutex sync.Mutex + topicNIDs map[string]int64 + insertTopicStmt *sql.Stmt + selectTopicStmt *sql.Stmt + selectTopicsStmt *sql.Stmt + insertMessageStmt *sql.Stmt + selectMessagesStmt *sql.Stmt + selectMaxOffsetStmt *sql.Stmt +} + +// NewPostgresqlDatabase creates a new naffka database using a postgresql database. +// Returns an error if there was a problem setting up the database. +func NewPostgresqlDatabase(db *sql.DB) (Database, error) { + var err error + + p := &postgresqlDatabase{ + db: db, + topicNIDs: map[string]int64{}, + } + + if _, err = db.Exec(postgresqlSchema); err != nil { + return nil, err + } + + for _, s := range []struct { + sql string + stmt **sql.Stmt + }{ + {insertTopicSQL, &p.insertTopicStmt}, + {selectTopicSQL, &p.selectTopicStmt}, + {selectTopicsSQL, &p.selectTopicsStmt}, + {insertMessageSQL, &p.insertMessageStmt}, + {selectMessagesSQL, &p.selectMessagesStmt}, + {selectMaxOffsetSQL, &p.selectMaxOffsetStmt}, + } { + *s.stmt, err = db.Prepare(s.sql) + if err != nil { + return nil, err + } + } + return p, nil +} + +// StoreMessages implements Database. +func (p *postgresqlDatabase) StoreMessages(topic string, messages []Message) error { + // Store the messages inside a single database transaction. + return withTransaction(p.db, func(txn *sql.Tx) error { + s := txn.Stmt(p.insertMessageStmt) + topicNID, err := p.assignTopicNID(txn, topic) + if err != nil { + return err + } + for _, m := range messages { + _, err = s.Exec(topicNID, m.Offset, m.Key, m.Value, m.Timestamp.UnixNano()) + if err != nil { + return err + } + } + return nil + }) +} + +// FetchMessages implements Database. +func (p *postgresqlDatabase) FetchMessages(topic string, startOffset, endOffset int64) (messages []Message, err error) { + topicNID, err := p.getTopicNID(nil, topic) + if err != nil { + return + } + rows, err := p.selectMessagesStmt.Query(topicNID, startOffset, endOffset) + if err != nil { + return + } + defer rows.Close() + for rows.Next() { + var ( + offset int64 + key []byte + value []byte + timestampNano int64 + ) + if err = rows.Scan(&offset, &key, &value, ×tampNano); err != nil { + return + } + messages = append(messages, Message{ + Offset: offset, + Key: key, + Value: value, + Timestamp: time.Unix(0, timestampNano), + }) + } + return +} + +// MaxOffsets implements Database. +func (p *postgresqlDatabase) MaxOffsets() (map[string]int64, error) { + topicNames, err := p.selectTopics() + if err != nil { + return nil, err + } + result := map[string]int64{} + for topicName, topicNID := range topicNames { + // Lookup the maximum offset. + maxOffset, err := p.selectMaxOffset(topicNID) + if err != nil { + return nil, err + } + if maxOffset > -1 { + // Don't include the topic if we haven't sent any messages on it. + result[topicName] = maxOffset + } + // Prefill the numeric ID cache. + p.addTopicNIDToCache(topicName, topicNID) + } + return result, nil +} + +// selectTopics fetches the names and numeric IDs for all the topics the +// database is aware of. +func (p *postgresqlDatabase) selectTopics() (map[string]int64, error) { + rows, err := p.selectTopicsStmt.Query() + if err != nil { + return nil, err + } + defer rows.Close() + result := map[string]int64{} + for rows.Next() { + var ( + topicName string + topicNID int64 + ) + if err = rows.Scan(&topicName, &topicNID); err != nil { + return nil, err + } + result[topicName] = topicNID + } + return result, nil +} + +// selectMaxOffset selects the maximum offset for a topic. +// Returns -1 if there aren't any messages for that topic. +// Returns an error if there was a problem talking to the database. +func (p *postgresqlDatabase) selectMaxOffset(topicNID int64) (maxOffset int64, err error) { + err = p.selectMaxOffsetStmt.QueryRow(topicNID).Scan(&maxOffset) + if err == sql.ErrNoRows { + return -1, nil + } + return maxOffset, err +} + +// getTopicNID finds the numeric ID for a topic. +// The txn argument is optional, this can be used outside a transaction +// by setting the txn argument to nil. +func (p *postgresqlDatabase) getTopicNID(txn *sql.Tx, topicName string) (topicNID int64, err error) { + // Get from the cache. + topicNID = p.getTopicNIDFromCache(topicName) + if topicNID != 0 { + return topicNID, nil + } + // Get from the database + s := p.selectTopicStmt + if txn != nil { + s = txn.Stmt(s) + } + err = s.QueryRow(topicName).Scan(&topicNID) + if err == sql.ErrNoRows { + return 0, nil + } + if err != nil { + return 0, err + } + // Update the shared cache. + p.addTopicNIDToCache(topicName, topicNID) + return topicNID, nil +} + +// assignTopicNID assigns a new numeric ID to a topic. +// The txn argument is mandatory, this is always called inside a transaction. +func (p *postgresqlDatabase) assignTopicNID(txn *sql.Tx, topicName string) (topicNID int64, err error) { + // Check if we already have a numeric ID for the topic name. + topicNID, err = p.getTopicNID(txn, topicName) + if err != nil { + return 0, err + } + if topicNID != 0 { + return topicNID, err + } + // We don't have a numeric ID for the topic name so we add an entry to the + // topics table. If the insert stmt succeeds then it will return the ID. + err = txn.Stmt(p.insertTopicStmt).QueryRow(topicName).Scan(&topicNID) + if err == sql.ErrNoRows { + // If the insert stmt succeeded, but didn't return any rows then it + // means that someone has added a row for the topic name between us + // selecting it the first time and us inserting our own row. + // (N.B. postgres only returns modified rows when using "RETURNING") + // So we can now just select the row that someone else added. + // TODO: This is probably unnecessary since naffka writes to a topic + // from a single thread. + return p.getTopicNID(txn, topicName) + } + if err != nil { + return 0, err + } + // Update the cache. + p.addTopicNIDToCache(topicName, topicNID) + return topicNID, nil +} + +// getTopicNIDFromCache returns the topicNID from the cache or returns 0 if the +// topic is not in the cache. +func (p *postgresqlDatabase) getTopicNIDFromCache(topicName string) (topicNID int64) { + p.topicsMutex.Lock() + defer p.topicsMutex.Unlock() + return p.topicNIDs[topicName] +} + +// addTopicNIDToCache adds the numeric ID for the topic to the cache. +func (p *postgresqlDatabase) addTopicNIDToCache(topicName string, topicNID int64) { + p.topicsMutex.Lock() + defer p.topicsMutex.Unlock() + p.topicNIDs[topicName] = topicNID +} + +// withTransaction runs a block of code passing in an SQL transaction +// If the code returns an error or panics then the transactions is rolledback +// Otherwise the transaction is committed. +func withTransaction(db *sql.DB, fn func(txn *sql.Tx) error) (err error) { + txn, err := db.Begin() + if err != nil { + return + } + defer func() { + if r := recover(); r != nil { + txn.Rollback() + panic(r) + } else if err != nil { + txn.Rollback() + } else { + err = txn.Commit() + } + }() + err = fn(txn) + return +} From bb8dcb09a1b0bb38ea1e834e739fe5415ff3ec6b Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@googlemail.com> Date: Fri, 17 Nov 2017 10:27:28 +0000 Subject: [PATCH 27/29] use voip turnServers struct from gomatrix for dedup and consistency (#344) Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .../matrix-org/dendrite/clientapi/routing/voip.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go b/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go index c4c48bd73..1a2a87527 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go @@ -26,16 +26,10 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/gomatrix" "github.com/matrix-org/util" ) -type turnServerResponse struct { - Username string `json:"username"` - Password string `json:"password"` - URIs []string `json:"uris"` - TTL int `json:"ttl"` -} - // RequestTurnServer implements: // GET /voip/turnServer func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg config.Dendrite) util.JSONResponse { @@ -52,7 +46,7 @@ func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg config.D // Duration checked at startup, err not possible duration, _ := time.ParseDuration(turnConfig.UserLifetime) - resp := turnServerResponse{ + resp := gomatrix.RespTurnServer{ URIs: turnConfig.URIs, TTL: int(duration.Seconds()), } From 19a716e7da3ac8ec5867cc85b7e93d439f77d7e0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 18 Nov 2017 15:06:51 +0000 Subject: [PATCH 28/29] Fix create filter API (#342) --- .../dendrite/clientapi/auth/storage/accounts/filter_table.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go index 9e3b7d6e6..81bae4545 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go @@ -92,11 +92,11 @@ func (s *filterStatements) insertFilter( // Check if filter already exists in the database err = s.selectFilterIDByContentStmt.QueryRowContext(ctx, localpart, filterJSON).Scan(&existingFilterID) - if err != nil { + if err != nil && err != sql.ErrNoRows { return "", err } // If it does, return the existing ID - if len(existingFilterID) != 0 { + if existingFilterID != "" { return existingFilterID, err } From ea53558cca358f0b72141922ab37a8b41ee775f6 Mon Sep 17 00:00:00 2001 From: Ross Schulman Date: Mon, 20 Nov 2017 09:33:49 -0500 Subject: [PATCH 29/29] Implement room_alias federation end point (#338) * Add room alias query endpoint * Try to fix indentation problems * Fix linting errors and use of httpReq.FormValue Signed-off-by: Ross Schulman * Run gofmt * Check for empty alias parameter and fix route URL Signed-off-by: Ross Schulman * Fix some linting errors Signed-off-by: Ross Schulman * Delete extra copy of directory route --- .../dendrite-federation-api-server/main.go | 3 +- .../cmd/dendrite-monolith-server/main.go | 2 +- .../dendrite/federationapi/routing/query.go | 96 +++++++++++++++++++ .../dendrite/federationapi/routing/routing.go | 10 ++ 4 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 src/github.com/matrix-org/dendrite/federationapi/routing/query.go diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go index ba981d8cf..53587ee20 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go @@ -80,6 +80,7 @@ func main() { queryAPI := api.NewRoomserverQueryAPIHTTP(cfg.RoomServerURL(), nil) inputAPI := api.NewRoomserverInputAPIHTTP(cfg.RoomServerURL(), nil) + aliasAPI := api.NewRoomserverAliasAPIHTTP(cfg.RoomServerURL(), nil) roomserverProducer := producers.NewRoomserverProducer(inputAPI) @@ -90,7 +91,7 @@ func main() { log.Info("Starting federation API server on ", cfg.Listen.FederationAPI) api := mux.NewRouter() - routing.Setup(api, *cfg, queryAPI, roomserverProducer, keyRing, federation, accountDB) + routing.Setup(api, *cfg, queryAPI, aliasAPI, roomserverProducer, keyRing, federation, accountDB) common.SetupHTTPAPI(http.DefaultServeMux, api) log.Fatal(http.ListenAndServe(string(cfg.Listen.FederationAPI), nil)) diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go index fafd4cb82..05fc4252b 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go @@ -348,7 +348,7 @@ func (m *monolith) setupAPIs() { ), m.syncAPIDB, m.deviceDB) federationapi_routing.Setup( - m.api, *m.cfg, m.queryAPI, m.roomServerProducer, m.keyRing, m.federation, + m.api, *m.cfg, m.queryAPI, m.aliasAPI, m.roomServerProducer, m.keyRing, m.federation, m.accountDB, ) diff --git a/src/github.com/matrix-org/dendrite/federationapi/routing/query.go b/src/github.com/matrix-org/dendrite/federationapi/routing/query.go new file mode 100644 index 000000000..ef4b8961a --- /dev/null +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/query.go @@ -0,0 +1,96 @@ +// Copyright 2017 New Vector Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "fmt" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrix" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// RoomAliasToID converts the queried alias into a room ID and returns it +func RoomAliasToID( + httpReq *http.Request, + federation *gomatrixserverlib.FederationClient, + cfg config.Dendrite, + aliasAPI api.RoomserverAliasAPI, +) util.JSONResponse { + roomAlias := httpReq.FormValue("alias") + if roomAlias == "" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("Must supply room alias parameter."), + } + } + _, domain, err := gomatrixserverlib.SplitID('#', roomAlias) + if err != nil { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"), + } + } + + var resp gomatrixserverlib.RespDirectory + + if domain == cfg.Matrix.ServerName { + queryReq := api.GetAliasRoomIDRequest{Alias: roomAlias} + var queryRes api.GetAliasRoomIDResponse + if err = aliasAPI.GetAliasRoomID(httpReq.Context(), &queryReq, &queryRes); err != nil { + return httputil.LogThenError(httpReq, err) + } + + if queryRes.RoomID == "" { + // TODO: List servers that are aware of this room alias + resp = gomatrixserverlib.RespDirectory{ + RoomID: queryRes.RoomID, + Servers: []gomatrixserverlib.ServerName{}, + } + } else { + // If the response doesn't contain a non-empty string, return an error + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound(fmt.Sprintf("Room alias %s not found", roomAlias)), + } + } + } else { + resp, err = federation.LookupRoomAlias(httpReq.Context(), domain, roomAlias) + if err != nil { + switch x := err.(type) { + case gomatrix.HTTPError: + if x.Code == 404 { + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound("Room alias not found"), + } + } + } + // TODO: Return 502 if the remote server errored. + // TODO: Return 504 if the remote server timed out. + return httputil.LogThenError(httpReq, err) + } + } + + return util.JSONResponse{ + Code: 200, + JSON: resp, + } +} diff --git a/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go b/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go index c50afd6e0..b23d738f9 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go @@ -38,6 +38,7 @@ func Setup( apiMux *mux.Router, cfg config.Dendrite, query api.RoomserverQueryAPI, + aliasAPI api.RoomserverAliasAPI, producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, @@ -105,6 +106,15 @@ func Setup( }, )).Methods("GET") + v1fedmux.Handle("/query/directory/", common.MakeFedAPI( + "federation_query_room_alias", cfg.Matrix.ServerName, keys, + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { + return RoomAliasToID( + httpReq, federation, cfg, aliasAPI, + ) + }, + )).Methods("GET") + v1fedmux.Handle("/query/profile", common.MakeFedAPI( "federation_query_profile", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {