RSDK-13120: update protos

docs/examples/my_cool_arm.py

@@ -2,11 +2,11 @@
 
 import asyncio
 import json
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Mapping, Optional, Tuple, Union
 
 from viam.components.arm import Arm, JointPositions, KinematicsFileFormat, Pose
 from viam.operations import run_with_operation
-from viam.proto.common import Capsule, Geometry, Sphere
+from viam.proto.common import Capsule, Geometry, Mesh, Sphere
 
 
 class MyCoolArm(Arm):
@@ -100,7 +100,9 @@ async def is_moving(self) -> bool:
     async def get_geometries(self, *, extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None) -> List[Geometry]:
         return self.geometries
 
-    async def get_kinematics(self, extra: Optional[Dict[str, Any]] = None, **kwargs) -> Tuple[KinematicsFileFormat.ValueType, bytes]:
+    async def get_kinematics(
+        self, extra: Optional[Dict[str, Any]] = None, **kwargs
+    ) -> Union[Tuple[KinematicsFileFormat.ValueType, bytes], Tuple[KinematicsFileFormat.ValueType, bytes, Mapping[str, Mesh]]]:
         return KinematicsFileFormat.KINEMATICS_FILE_FORMAT_SVA, self.kinematics
 
     async def close(self):

examples/server/v1/components.py

@@ -19,7 +19,6 @@
 from PIL import Image
 
 from viam.components.arm import Arm
-from viam.components.audio_in import AudioIn, AudioResponse
 from viam.components.audio_out import AudioOut
 from viam.components.base import Base
 from viam.components.board import Board, TickStream
@@ -48,6 +47,7 @@
     ResponseMetadata,
     Sphere,
     Vector3,
+    Mesh,
 )
 from viam.proto.component.arm import JointPositions
 from viam.proto.component.encoder import PositionType
@@ -73,7 +73,7 @@ def __init__(self, name: str):
         )
         self.joint_positions = JointPositions(values=[0, 0, 0, 0, 0, 0])
         self.is_stopped = True
-        self.kinematics = (KinematicsFileFormat.KINEMATICS_FILE_FORMAT_SVA, b"\x00\x01\x02")
+        self.kinematics = (KinematicsFileFormat.KINEMATICS_FILE_FORMAT_SVA, b"\x00\x01\x02", {})
         super().__init__(name)
 
     async def get_end_position(self, extra: Optional[Dict[str, Any]] = None, **kwargs) -> Pose:
@@ -101,71 +101,12 @@ async def stop(self, extra: Optional[Dict[str, Any]] = None, **kwargs):
     async def is_moving(self):
         return not self.is_stopped
 
-    async def get_kinematics(self, extra: Optional[Dict[str, Any]] = None, **kwargs) -> Tuple[KinematicsFileFormat.ValueType, bytes]:
+    async def get_kinematics(self, extra: Optional[Dict[str, Any]] = None, **kwargs) -> Tuple[KinematicsFileFormat.ValueType, bytes, Mapping[str, Mesh]]:
         return self.kinematics
 
     async def get_geometries(self, extra: Optional[Dict[str, Any]] = None, **kwargs) -> List[Geometry]:
         return GEOMETRIES
 
-class ExampleAudioIn(AudioIn):
-    def __init__(self, name: str):
-        super().__init__(name)
-        self.sample_rate = 44100
-        self.num_channels = 2
-        self.supported_codecs = ["pcm16"]
-        self.chunk_count = 0
-        self.latency = timedelta(milliseconds=20)
-        self.volume_scale = 0.2
-        self.frequency_hz = 440
-
-    async def get_audio(
-        self, codec: str, duration_seconds: float, previous_timestamp_ns: int, *, timeout: Optional[float] = None, **kwargs
-    ) -> AudioIn.AudioStream:
-        async def read() -> AsyncIterator[AudioIn.AudioResponse]:
-            # Generate chunks based on duration
-            chunk_duration_ms = 100  # 100ms per chunk
-            chunks_to_generate = max(1, int((duration_seconds * 1000) / chunk_duration_ms))
-
-            for i in range(chunks_to_generate):
-                # Generate audio data (sine wave pattern)
-                chunk_data = b""
-                samples_per_chunk = int(self.sample_rate * (chunk_duration_ms / 1000))
-
-                for sample in range(samples_per_chunk):
-                    # Calculate the timing in seconds of this audio sample
-                    time_offset = (i * chunk_duration_ms / 1000) + (sample / self.sample_rate)
-                    # Generate one 16-bit PCM audio sample for a sine wave
-                    # 32767 scales the value from (-1,1) to full 16 bit signed range (-32768,32767)
-                    amplitude = int(32767 * self.volume_scale * math.sin(2 * math.pi * self.frequency_hz * time_offset))
-
-                    # Convert to 16-bit PCM stereo
-                    sample_bytes = amplitude.to_bytes(2, byteorder="little", signed=True)
-                    chunk_data += sample_bytes * self.num_channels
-
-                chunk_start_time = previous_timestamp_ns + (i * chunk_duration_ms * 1000000)  # Convert ms to ns
-                chunk_end_time = chunk_start_time + (chunk_duration_ms * 1000000)
-
-                audio_chunk = AudioInChunk(
-                    audio_data=bytes(chunk_data),
-                    audio_info=AudioInfo(codec=codec, sample_rate_hz=int(self.sample_rate), num_channels=self.num_channels),
-                    sequence=i,
-                    start_timestamp_nanoseconds=chunk_start_time,
-                    end_timestamp_nanoseconds=chunk_end_time,
-                )
-                audio_response = AudioResponse(audio=audio_chunk)
-                yield audio_response
-
-                await asyncio.sleep(self.latency.total_seconds())
-
-        return StreamWithIterator(read())
-
-    async def get_properties(self, *, timeout: Optional[float] = None, **kwargs) -> AudioIn.Properties:
-        """Return the audio input device properties."""
-        return AudioIn.Properties(supported_codecs=self.supported_codecs, sample_rate_hz=self.sample_rate, num_channels=self.num_channels)
-
-    async def get_geometries(self, extra: Optional[Dict[str, Any]] = None, **kwargs) -> List[Geometry]:
-        return GEOMETRIES
-
 
 class ExampleAudioOut(AudioOut):
     def __init__(self, name: str):
@@ -546,16 +487,16 @@ async def is_moving(self):
     async def get_geometries(self, extra: Optional[Dict[str, Any]] = None, **kwargs) -> List[Geometry]:
         return GEOMETRIES
 
-    async def get_kinematics(self, *, extra=None, timeout=None, **kwargs) -> Tuple[KinematicsFileFormat.ValueType, bytes]:
-        return (KinematicsFileFormat.KINEMATICS_FILE_FORMAT_UNSPECIFIED, b"abc")
+    async def get_kinematics(self, *, extra=None, timeout=None, **kwargs) -> Tuple[KinematicsFileFormat.ValueType, bytes, Mapping[str, Mesh]]:
+        return (KinematicsFileFormat.KINEMATICS_FILE_FORMAT_UNSPECIFIED, b"abc", {})
 
 
 class ExampleGripper(Gripper):
     def __init__(self, name: str):
         self.opened = False
         self.is_stopped = True
         self.holding_something = False
-        self.kinematics = (KinematicsFileFormat.KINEMATICS_FILE_FORMAT_SVA, b"\x00\x01\x02")
+        self.kinematics = (KinematicsFileFormat.KINEMATICS_FILE_FORMAT_SVA, b"\x00\x01\x02", {})
         super().__init__(name)
 
     async def open(self, extra: Optional[Dict[str, Any]] = None, **kwargs):
@@ -581,7 +522,7 @@ async def is_moving(self):
     async def get_geometries(self, extra: Optional[Dict[str, Any]] = None, **kwargs) -> List[Geometry]:
         return GEOMETRIES
 
-    async def get_kinematics(self, extra: Optional[Dict[str, Any]] = None, **kwargs) -> Tuple[KinematicsFileFormat.ValueType, bytes]:
+    async def get_kinematics(self, extra: Optional[Dict[str, Any]] = None, **kwargs) -> Tuple[KinematicsFileFormat.ValueType, bytes, Mapping[str, Mesh]]:
         return self.kinematics
 
 

examples/server/v1/server.py

@@ -8,7 +8,6 @@
 from .components import (
     ExampleAnalog,
     ExampleArm,
-    ExampleAudioIn,
     ExampleAudioOut,
     ExampleBase,
     ExampleBoard,
@@ -30,7 +29,6 @@
 
 async def run(host: str, port: int, log_level: int):
     my_arm = ExampleArm("arm0")
-    my_audio_in = ExampleAudioIn("audio_in0")
     my_audio_out = ExampleAudioOut("audio_out0")
     my_base = ExampleBase("base0")
     my_board = ExampleBoard(
@@ -77,7 +75,6 @@ async def run(host: str, port: int, log_level: int):
     server = Server(
         resources=[
             my_arm,
-            my_audio_in,
             my_audio_out,
             my_base,
             my_board,

src/viam/app/app_client.py

@@ -165,6 +165,8 @@
     UpdateRobotResponse,
     UploadModuleFileRequest,
     Visibility,
+    FragmentImport,
+    FragmentImportList,
 )
 from viam.proto.app import Fragment as FragmentPB
 from viam.proto.app import FragmentHistoryEntry as FragmentHistoryEntryPB
@@ -736,6 +738,7 @@ async def update_organization(
         public_namespace: Optional[str] = None,
         region: Optional[str] = None,
         cid: Optional[str] = None,
+        default_fragments: Optional[List[FragmentImport]] = None,
     ) -> Organization:
         """Updates organization details.
 
@@ -768,6 +771,11 @@ async def update_organization(
             region=region,
             cid=cid,
             name=name,
+            default_fragments=(
+                FragmentImportList(fragments=default_fragments)
+                if default_fragments is not None
+                else None
+            ),
         )
         response: UpdateOrganizationResponse = await self._app_client.UpdateOrganization(request, metadata=self._metadata)
         return response.organization

src/viam/app/data_client.py

@@ -1535,6 +1535,7 @@ async def binary_data_capture_upload(
         tags: Optional[List[str]] = None,
         dataset_ids: Optional[List[str]] = None,
         data_request_times: Optional[Tuple[datetime, datetime]] = None,
+        mime_type: Optional[str] = None,
     ) -> str:
         """Upload binary sensor data.
 
@@ -1573,6 +1574,7 @@ async def binary_data_capture_upload(
             dataset_ids (Optional[List[str]]): Optional list of datasets to add the data to.
             data_request_times (Optional[Tuple[datetime.datetime, datetime.datetime]]): Optional tuple containing datetime objects
                 denoting the times this data was requested ``[0]`` by the robot and received ``[1]`` from the appropriate sensor.
+            mime_type (Optional[str]): Optional mime type of the data.
 
         Raises:
             GRPCError: If an invalid part ID is passed.
@@ -1603,6 +1605,7 @@ async def binary_data_capture_upload(
             method_parameters=method_parameters,
             tags=tags,
             dataset_ids=dataset_ids,
+            mime_type=mime_type or "",
         )
         if file_extension:
             metadata.file_extension = file_extension if file_extension[0] == "." else f".{file_extension}"
@@ -1721,6 +1724,7 @@ async def streaming_data_capture_upload(
         data_request_times: Optional[Tuple[datetime, datetime]] = None,
         tags: Optional[List[str]] = None,
         dataset_ids: Optional[List[str]] = None,
+        mime_type: Optional[str] = None,
     ) -> str:
         """Uploads the metadata and contents of streaming binary data.
 
@@ -1753,6 +1757,7 @@ async def streaming_data_capture_upload(
                 denoting the times this data was requested ``[0]`` by the robot and received ``[1]`` from the appropriate sensor.
             tags (Optional[List[str]]): Optional list of tags to allow for tag-based filtering when retrieving data.
             dataset_ids (Optional[List[str]]): Optional list of datasets to add the data to.
+            mime_type (Optional[str]): Optional mime type of the data.
 
         Raises:
             GRPCError: If an invalid part ID is passed.
@@ -1773,6 +1778,7 @@ async def streaming_data_capture_upload(
             file_extension=file_ext if file_ext[0] == "." else f".{file_ext}",
             tags=tags,
             dataset_ids=dataset_ids,
+            mime_type=mime_type or "",
         )
         sensor_metadata = SensorMetadata(
             time_requested=datetime_to_timestamp(data_request_times[0]) if data_request_times else None,
@@ -1802,6 +1808,7 @@ async def file_upload(
         file_extension: Optional[str] = None,
         tags: Optional[List[str]] = None,
         dataset_ids: Optional[List[str]] = None,
+        mime_type: Optional[str] = None,
     ) -> str:
         """Upload arbitrary file data.
 
@@ -1832,6 +1839,7 @@ async def file_upload(
                 isn't provided. Files with a ``.jpeg``, ``.jpg``, or ``.png`` extension will be saved to the **Images** tab.
             tags (Optional[List[str]]): Optional list of tags to allow for tag-based filtering when retrieving data.
             dataset_ids (Optional[List[str]]): Optional list of datasets to add the data to.
+            mime_type (Optional[str]): Optional mime type of the data.
 
         Raises:
             GRPCError: If an invalid part ID is passed.
@@ -1866,6 +1874,7 @@ async def file_upload_from_path(
         method_parameters: Optional[Mapping[str, Any]] = None,
         tags: Optional[List[str]] = None,
         dataset_ids: Optional[List[str]] = None,
+        mime_type: Optional[str] = None,
     ) -> str:
         """Upload arbitrary file data.
 
@@ -1890,6 +1899,7 @@ async def file_upload_from_path(
             method_parameters (Optional[str]): Optional dictionary of the method parameters. No longer in active use.
             tags (Optional[List[str]]): Optional list of tags to allow for tag-based filtering when retrieving data.
             dataset_ids (Optional[List[str]]): Optional list of datasets to add the data to.
+            mime_type (Optional[str]): Optional mime type of the data.
 
         Raises:
             GRPCError: If an invalid part ID is passed.
@@ -1917,6 +1927,7 @@ async def file_upload_from_path(
             file_extension=file_extension if file_extension else "",
             tags=tags,
             dataset_ids=dataset_ids,
+            mime_type=mime_type or "",
         )
         response: FileUploadResponse = await self._file_upload(metadata=metadata, file_contents=FileData(data=data if data else bytes()))
         return response.binary_data_id

src/viam/components/__init__.py

@@ -0,0 +1,12 @@
+from typing import Mapping, Tuple, Union
+
+from viam.proto.common import KinematicsFileFormat, Mesh
+
+KinematicsReturn = Union[
+    Tuple[KinematicsFileFormat.ValueType, bytes],
+    Tuple[KinematicsFileFormat.ValueType, bytes, Mapping[str, Mesh]],
+]
+
+__all__ = [
+    "KinematicsReturn",
+]

src/viam/components/arm/__init__.py

@@ -1,3 +1,4 @@
+from viam.components import KinematicsReturn
 from viam.proto.common import KinematicsFileFormat, Pose
 from viam.proto.component.arm import JointPositions
 from viam.resource.registry import Registry, ResourceRegistration
@@ -10,6 +11,7 @@
     "Arm",
     "JointPositions",
     "KinematicsFileFormat",
+    "KinematicsReturn",
     "Pose",
 ]
 

src/viam/components/arm/arm.py

@@ -1,10 +1,11 @@
 import abc
-from typing import Any, Dict, Final, Optional, Tuple
+from typing import Any, Dict, Final, Optional
 
 from viam.resource.types import API, RESOURCE_NAMESPACE_RDK, RESOURCE_TYPE_COMPONENT
 
+from .. import KinematicsReturn
 from ..component_base import ComponentBase
-from . import JointPositions, KinematicsFileFormat, Pose
+from . import JointPositions, Pose
 
 
 class Arm(ComponentBase):
@@ -195,7 +196,7 @@ async def is_moving(self) -> bool:
     @abc.abstractmethod
     async def get_kinematics(
         self, *, extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **kwargs
-    ) -> Tuple[KinematicsFileFormat.ValueType, bytes]:
+    ) -> KinematicsReturn:
         """
         Get the kinematics information associated with the arm.
 
@@ -217,6 +218,7 @@ async def get_kinematics(
             file, either in URDF format (``KinematicsFileFormat.KINEMATICS_FILE_FORMAT_URDF``) or
             Viam's kinematic parameter format (spatial vector algebra) (``KinematicsFileFormat.KINEMATICS_FILE_FORMAT_SVA``),
             and the second [1] value represents the byte contents of the file.
+            If available, a third [2] value provides meshes keyed by URDF filepath.
 
         For more information, see `Arm component <https://docs.viam.com/dev/reference/apis/components/arm/#getkinematics>`_.
         """

src/viam/components/arm/client.py

@@ -1,7 +1,8 @@
-from typing import Any, Dict, List, Mapping, Optional, Tuple
+from typing import Any, Dict, List, Mapping, Optional
 
 from grpclib.client import Channel
 
+from viam.components import KinematicsReturn
 from viam.proto.common import DoCommandRequest, DoCommandResponse, Geometry, GetKinematicsRequest, GetKinematicsResponse
 from viam.proto.component.arm import (
     ArmServiceStub,
@@ -19,7 +20,7 @@
 from viam.resource.rpc_client_base import ReconfigurableResourceRPCClientBase
 from viam.utils import ValueTypes, dict_to_struct, get_geometries, struct_to_dict
 
-from . import Arm, KinematicsFileFormat, Pose
+from . import Arm, Pose
 
 
 class ArmClient(Arm, ReconfigurableResourceRPCClientBase):
@@ -113,11 +114,11 @@ async def do_command(
 
     async def get_kinematics(
         self, *, extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **kwargs
-    ) -> Tuple[KinematicsFileFormat.ValueType, bytes]:
+    ) -> KinematicsReturn:
         md = kwargs.get("metadata", self.Metadata()).proto
         request = GetKinematicsRequest(name=self.name, extra=dict_to_struct(extra))
         response: GetKinematicsResponse = await self.client.GetKinematics(request, timeout=timeout, metadata=md)
-        return (response.format, response.kinematics_data)
+        return (response.format, response.kinematics_data, response.meshes_by_urdf_filepath)
 
     async def get_geometries(self, *, extra: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, **kwargs) -> List[Geometry]:
         md = kwargs.get("metadata", self.Metadata())

src/viam/components/arm/service.py

@@ -109,8 +109,13 @@ async def GetKinematics(self, stream: Stream[GetKinematicsRequest, GetKinematics
         assert request is not None
         arm = self.get_resource(request.name)
         timeout = stream.deadline.time_remaining() if stream.deadline else None
-        format, kinematics_data = await arm.get_kinematics(extra=struct_to_dict(request.extra), timeout=timeout, metadata=stream.metadata)
-        response = GetKinematicsResponse(format=format, kinematics_data=kinematics_data)
+        kinematics = await arm.get_kinematics(extra=struct_to_dict(request.extra), timeout=timeout, metadata=stream.metadata)
+        if len(kinematics) == 2:
+            format, kinematics_data = kinematics
+            meshes = {}
+        else:
+            format, kinematics_data, meshes = kinematics
+        response = GetKinematicsResponse(format=format, kinematics_data=kinematics_data, meshes_by_urdf_filepath=meshes)
         await stream.send_message(response)
 
     async def GetGeometries(self, stream: Stream[GetGeometriesRequest, GetGeometriesResponse]) -> None: