diff --git a/Programs/Simian/Agent.cs b/Programs/Simian/Agent.cs
index 1bae2909..6ebb3d7e 100644
--- a/Programs/Simian/Agent.cs
+++ b/Programs/Simian/Agent.cs
@@ -15,6 +15,8 @@ namespace Simian
         public uint CircuitCode;
         public string FirstName;
         public string LastName;
+        public Avatar Avatar = new Avatar();
+        public AgentManager.ControlFlags ControlFlags = AgentManager.ControlFlags.NONE;
 
         /// <summary>Sequence numbers of packets we've received (for duplicate checking)</summary>
         internal Queue<uint> packetArchive = new Queue<uint>();
diff --git a/Programs/Simian/Extensions/SceneManager.cs b/Programs/Simian/Extensions/SceneManager.cs
index 4bac3009..23b23bc3 100644
--- a/Programs/Simian/Extensions/SceneManager.cs
+++ b/Programs/Simian/Extensions/SceneManager.cs
@@ -45,9 +45,8 @@ namespace Simian
             avatar.Rotation = Quaternion.Identity;
             avatar.Scale = new Vector3(1f, 1f, 3f);
 
-            // Add this agent to the scene graph
-            lock (server.SceneAvatars)
-                server.SceneAvatars[avatar.ID] = avatar;
+            // Link this avatar up with the corresponding agent
+            agent.Avatar = avatar;
 
             AgentMovementCompletePacket complete = new AgentMovementCompletePacket();
             complete.AgentData.AgentID = agent.AgentID;
@@ -65,11 +64,7 @@ namespace Simian
         {
             AgentUpdatePacket update = (AgentUpdatePacket)packet;
 
-            Avatar avatar;
-            if (server.SceneAvatars.TryGetValue(agent.AgentID, out avatar))
-            {
-                SendFullUpdate(agent, avatar, update.AgentData.State, update.AgentData.Flags);
-            }
+            SendFullUpdate(agent, agent.Avatar, update.AgentData.State, update.AgentData.Flags);
         }
 
         void AgentWearablesRequestHandler(Packet packet, Agent agent)
diff --git a/Programs/Simian/Simian.cs b/Programs/Simian/Simian.cs
index 73bfab8a..1024cc9c 100644
--- a/Programs/Simian/Simian.cs
+++ b/Programs/Simian/Simian.cs
@@ -16,7 +16,9 @@ namespace Simian
         public HttpServer HttpServer;
         public UDPServer UDPServer;
         public Dictionary<uint, Primitive> SceneObjects = new Dictionary<uint, Primitive>();
-        public Dictionary<UUID, Avatar> SceneAvatars = new Dictionary<UUID, Avatar>();
+
+        /// <summary>All of the agents currently connected to this UDP server</summary>
+        public Dictionary<IPEndPoint, Agent> Agents = new Dictionary<IPEndPoint, Agent>();
 
         Dictionary<uint, Agent> unassociatedAgents;
         int currentCircuitCode;
diff --git a/Programs/Simian/UDPServer.cs b/Programs/Simian/UDPServer.cs
index 73b55b08..c2a6a8a6 100644
--- a/Programs/Simian/UDPServer.cs
+++ b/Programs/Simian/UDPServer.cs
@@ -27,8 +27,6 @@ namespace Simian
         Simian server;
         /// <summary>Handlers for incoming packets</summary>
         PacketEventDictionary packetEvents = new PacketEventDictionary();
-        /// <summary>All of the agents currently connected to this UDP server</summary>
-        Dictionary<IPEndPoint, Agent> agents = new Dictionary<IPEndPoint, Agent>();
         /// <summary>Incoming packets that are awaiting handling</summary>
         BlockingQueue<IncomingPacket> packetInbox = new BlockingQueue<IncomingPacket>(Settings.PACKET_INBOX_SIZE);
 
@@ -84,8 +82,8 @@ namespace Simian
                 {
                     agent.Initialize((IPEndPoint)buffer.RemoteEndPoint);
 
-                    lock (agents)
-                        agents[(IPEndPoint)buffer.RemoteEndPoint] = agent;
+                    lock (server.Agents)
+                        server.Agents[(IPEndPoint)buffer.RemoteEndPoint] = agent;
 
                     Logger.Log("Activated UDP circuit " + useCircuitCode.CircuitCode.Code, Helpers.LogLevel.Info);
 
@@ -101,7 +99,7 @@ namespace Simian
             else
             {
                 // Determine which agent this packet came from
-                if (!agents.TryGetValue((IPEndPoint)buffer.RemoteEndPoint, out agent))
+                if (!server.Agents.TryGetValue((IPEndPoint)buffer.RemoteEndPoint, out agent))
                 {
                     Logger.Log("Received UDP packet from an unrecognized source: " + ((IPEndPoint)buffer.RemoteEndPoint).ToString(),
                         Helpers.LogLevel.Warning);