master: Fixed 对讲问题修复

src/utils/Intercom.ts

@@ -6,10 +6,10 @@ import type {
   // ChatMessage,
   RoomConnectOptions,
   RoomOptions,
-  // ScalabilityMode,
+  ScalabilityMode,
   // SimulationScenario,
   // VideoCaptureOptions,
-  // VideoCodec,
+  VideoCodec,
 } from 'livekit-client';
 import {
   ConnectionQuality,
@@ -17,25 +17,25 @@ import {
   DisconnectReason,
   ExternalE2EEKeyProvider,
   LocalAudioTrack,
-  // LocalParticipant,
+  LocalParticipant,
   LogLevel,
   MediaDeviceFailure,
   Participant,
   ParticipantEvent,
   RemoteParticipant,
-  // RemoteTrackPublication,
-  // RemoteVideoTrack,
+  RemoteTrackPublication,
+  RemoteVideoTrack,
   Room,
   RoomEvent,
   ScreenSharePresets,
-  // Track,
+  Track,
   TrackPublication,
-  // VideoPresets,
+  VideoPresets,
   // VideoQuality,
-  // createAudioAnalyser,
+  createAudioAnalyser,
   setLogLevel,
-  // supportsAV1,
-  // supportsVP9,
+  supportsAV1,
+  supportsVP9,
 } from 'livekit-client';
 import FingerprintJS from '@fingerprintjs/fingerprintjs';
 
@@ -73,6 +73,7 @@ const state = {
 let currentRoom: Room | undefined;
 
 let startTime: number;
+let timer: any;
 
 function appendLog(...args: any[]) {
   let logger = '';
@@ -89,6 +90,198 @@ function appendLog(...args: any[]) {
   console.log(logger);
 }
 
+function getParticipantsAreaElement() {
+  return (
+    (
+      window as { [k: string]: any }
+    ).documentPictureInPicture?.window?.document.querySelector(
+      '#participants-area'
+    ) || document.querySelector('#participants-area')
+  );
+}
+// updates participant UI
+function renderParticipant(participant: Participant, remove = false) {
+  const container = getParticipantsAreaElement();
+  if (!container) return;
+  const { identity } = participant;
+  let div = container.querySelector(`#participant-${identity}`);
+  if (!div && !remove) {
+    div = document.createElement('div');
+    div.id = `participant-${identity}`;
+    div.className = 'participant';
+    div.innerHTML = `
+      <video id="video-${identity}"></video>
+      <audio id="audio-${identity}"></audio>
+      <div class="info-bar">
+        <div id="name-${identity}" class="name">
+        </div>
+        <div style="text-align: center;">
+          <span id="codec-${identity}" class="codec">
+          </span>
+          <span id="size-${identity}" class="size">
+          </span>
+          <span id="bitrate-${identity}" class="bitrate">
+          </span>
+        </div>
+        <div class="right">
+          <span id="signal-${identity}"></span>
+          <span id="mic-${identity}" class="mic-on"></span>
+          <span id="e2ee-${identity}" class="e2ee-on"></span>
+        </div>
+      </div>
+      ${
+        participant instanceof RemoteParticipant
+          ? `<div class="volume-control">
+        <input id="volume-${identity}" type="range" min="0" max="1" step="0.1" value="1" orient="vertical" />
+      </div>`
+          : `<progress id="local-volume" max="1" value="0" />`
+      }
+
+    `;
+    container.appendChild(div);
+
+    const sizeElm = container.querySelector(`#size-${identity}`);
+    const videoElm = <HTMLVideoElement>(
+      container.querySelector(`#video-${identity}`)
+    );
+    // videoElm.onresize = () => {
+    //   updateVideoSize(videoElm!, sizeElm!);
+    // };
+  }
+  const videoElm = <HTMLVideoElement>(
+    container.querySelector(`#video-${identity}`)
+  );
+  const audioELm = <HTMLAudioElement>(
+    container.querySelector(`#audio-${identity}`)
+  );
+  if (remove) {
+    div?.remove();
+    if (videoElm) {
+      videoElm.srcObject = null;
+      videoElm.src = '';
+    }
+    if (audioELm) {
+      audioELm.srcObject = null;
+      audioELm.src = '';
+    }
+    return;
+  }
+
+  // update properties
+  const nameElement = container.querySelector(`#name-${identity}`);
+  if (nameElement) {
+    nameElement.innerHTML = participant.identity;
+    if (participant instanceof LocalParticipant) {
+      nameElement.innerHTML += ' (you)';
+    }
+  }
+  const micElm = container.querySelector(`#mic-${identity}`)!;
+  const signalElm = container.querySelector(`#signal-${identity}`)!;
+  const cameraPub = participant.getTrackPublication(Track.Source.Camera);
+  const micPub = participant.getTrackPublication(Track.Source.Microphone);
+  if (participant.isSpeaking) {
+    div!.classList.add('speaking');
+  } else {
+    div!.classList.remove('speaking');
+  }
+
+  if (participant instanceof RemoteParticipant) {
+    const volumeSlider = <HTMLInputElement>(
+      container.querySelector(`#volume-${identity}`)
+    );
+    volumeSlider.addEventListener('input', (ev) => {
+      participant.setVolume(
+        Number.parseFloat((ev.target as HTMLInputElement).value)
+      );
+    });
+  }
+
+  const cameraEnabled =
+    cameraPub && cameraPub.isSubscribed && !cameraPub.isMuted;
+  if (cameraEnabled) {
+    if (participant instanceof LocalParticipant) {
+      // flip
+      videoElm.style.transform = 'scale(-1, 1)';
+    } else if (!cameraPub?.videoTrack?.attachedElements.includes(videoElm)) {
+      const renderStartTime = Date.now();
+      // measure time to render
+      videoElm.onloadeddata = () => {
+        const elapsed = Date.now() - renderStartTime;
+        let fromJoin = 0;
+        if (
+          participant.joinedAt &&
+          participant.joinedAt.getTime() < startTime
+        ) {
+          fromJoin = Date.now() - startTime;
+        }
+        appendLog(
+          `RemoteVideoTrack ${cameraPub?.trackSid} (${videoElm.videoWidth}x${videoElm.videoHeight}) rendered in ${elapsed}ms`,
+          fromJoin > 0 ? `, ${fromJoin}ms from start` : ''
+        );
+      };
+    }
+    cameraPub?.videoTrack?.attach(videoElm);
+  } else {
+    // clear information display
+    const sizeElement = container.querySelector(`#size-${identity}`);
+    if (sizeElement) {
+      sizeElement.innerHTML = '';
+    }
+    if (cameraPub?.videoTrack) {
+      // detach manually whenever possible
+      cameraPub.videoTrack?.detach(videoElm);
+    } else {
+      videoElm.src = '';
+      videoElm.srcObject = null;
+    }
+  }
+
+  const micEnabled = micPub && micPub.isSubscribed && !micPub.isMuted;
+  if (micEnabled) {
+    if (!(participant instanceof LocalParticipant)) {
+      // don't attach local audio
+      audioELm.onloadeddata = () => {
+        if (
+          participant.joinedAt &&
+          participant.joinedAt.getTime() < startTime
+        ) {
+          const fromJoin = Date.now() - startTime;
+          appendLog(
+            `RemoteAudioTrack ${micPub?.trackSid} played ${fromJoin}ms from start`
+          );
+        }
+      };
+      micPub?.audioTrack?.attach(audioELm);
+    }
+    micElm.className = 'mic-on';
+    micElm.innerHTML = '<i class="fas fa-microphone"></i>';
+  } else {
+    micElm.className = 'mic-off';
+    micElm.innerHTML = '<i class="fas fa-microphone-slash"></i>';
+  }
+
+  const e2eeElm = container.querySelector(`#e2ee-${identity}`)!;
+  if (participant.isEncrypted) {
+    e2eeElm.className = 'e2ee-on';
+    e2eeElm.innerHTML = '<i class="fas fa-lock"></i>';
+  } else {
+    e2eeElm.className = 'e2ee-off';
+    e2eeElm.innerHTML = '<i class="fas fa-unlock"></i>';
+  }
+
+  switch (participant.connectionQuality) {
+    case ConnectionQuality.Excellent:
+    case ConnectionQuality.Good:
+    case ConnectionQuality.Poor:
+      signalElm.className = `connection-${participant.connectionQuality}`;
+      signalElm.innerHTML = '<i class="fas fa-circle"></i>';
+      break;
+    default:
+      signalElm.innerHTML = '';
+    // do nothing
+  }
+}
+
 function participantConnected(participant: Participant) {
   appendLog(
     'participant',
@@ -101,17 +294,21 @@ function participantConnected(participant: Participant) {
   participant
     .on(ParticipantEvent.TrackMuted, (pub: TrackPublication) => {
       appendLog('track was muted', pub.trackSid, participant.identity);
+      renderParticipant(participant);
     })
     .on(ParticipantEvent.TrackUnmuted, (pub: TrackPublication) => {
       appendLog('track was unmuted', pub.trackSid, participant.identity);
+      renderParticipant(participant);
     })
     .on(ParticipantEvent.IsSpeakingChanged, () => {
       // eslint-disable-next-line no-console
       console.log(participant);
+      renderParticipant(participant);
     })
     .on(ParticipantEvent.ConnectionQualityChanged, () => {
       // eslint-disable-next-line no-console
       console.log(participant);
+      renderParticipant(participant);
     });
 }
 
@@ -129,6 +326,77 @@ function handleRoomDisconnect(reason?: DisconnectReason) {
   (window as { [k: string]: any }).currentRoom = undefined;
 }
 
+const elementMapping: { [k: string]: MediaDeviceKind } = {
+  // 'video-input': 'videoinput',
+  'audio-input': 'audioinput',
+  'audio-output': 'audiooutput',
+} as const;
+
+async function handleDevicesChanged() {
+  Promise.all(
+    Object.keys(elementMapping).map(async (id) => {
+      const kind = elementMapping[id];
+      if (!kind) {
+        return;
+      }
+      const devices = await Room.getLocalDevices(kind);
+      // eslint-disable-next-line no-restricted-syntax
+      for (const device of devices) {
+        if (device.deviceId === state.defaultDevices.get(kind)) {
+          // eslint-disable-next-line no-console
+          console.log(device.label);
+        }
+      }
+    })
+  );
+}
+
+function renderScreenShare(room: Room) {
+  // const div = $('screenshare-area')!;
+  if (room.state !== ConnectionState.Connected) {
+    // div.style.display = 'none';
+    return;
+  }
+  let participant: Participant | undefined;
+  let screenSharePub: TrackPublication | undefined =
+    room.localParticipant.getTrackPublication(Track.Source.ScreenShare);
+  let screenShareAudioPub: RemoteTrackPublication | undefined;
+  if (!screenSharePub) {
+    room.remoteParticipants.forEach((p) => {
+      if (screenSharePub) {
+        return;
+      }
+      participant = p;
+      const pub = p.getTrackPublication(Track.Source.ScreenShare);
+      if (pub?.isSubscribed) {
+        screenSharePub = pub;
+      }
+      const audioPub = p.getTrackPublication(Track.Source.ScreenShareAudio);
+      if (audioPub?.isSubscribed) {
+        screenShareAudioPub = audioPub;
+      }
+    });
+  } else {
+    participant = room.localParticipant;
+  }
+
+  // if (screenSharePub && participant) {
+  //   // div.style.display = 'block';
+  //   // const videoElm = <HTMLVideoElement>$('screenshare-video');
+  //   screenSharePub.videoTrack?.attach(videoElm);
+  //   if (screenShareAudioPub) {
+  //     screenShareAudioPub.audioTrack?.attach(videoElm);
+  //   }
+  //   videoElm.onresize = () => {
+  //     updateVideoSize(videoElm, <HTMLSpanElement>$('screenshare-resolution'));
+  //   };
+  //   const infoElm = $('screenshare-info')!;
+  //   infoElm.innerHTML = `Screenshare from ${participant.identity}`;
+  // } else {
+  //   div.style.display = 'none';
+  // }
+}
+
 const connectToRoom = async (
   url: string,
   token: string,
@@ -158,7 +426,7 @@ const connectToRoom = async (
       const track = pub.track as LocalAudioTrack;
 
       if (track instanceof LocalAudioTrack) {
-        // const { calculateVolume } = createAudioAnalyser(track);
+        const { calculateVolume } = createAudioAnalyser(track);
         // setInterval(() => {
         //   $('local-volume')?.setAttribute(
         //     'value',
@@ -166,10 +434,15 @@ const connectToRoom = async (
         //   );
         // }, 200);
       }
+      renderScreenShare(room);
+    })
+    .on(RoomEvent.LocalTrackUnpublished, () => {
+      renderScreenShare(room);
     })
     .on(RoomEvent.RoomMetadataChanged, (metadata) => {
       appendLog('new metadata for room', metadata);
     })
+    .on(RoomEvent.MediaDevicesChanged, handleDevicesChanged)
     .on(RoomEvent.MediaDevicesError, (e: Error) => {
       const failure = MediaDeviceFailure.getFailure(e);
       appendLog('media device failure', failure);
@@ -182,9 +455,13 @@ const connectToRoom = async (
     )
     .on(RoomEvent.TrackSubscribed, (track, pub, participant) => {
       appendLog('subscribed to track', pub.trackSid, participant.identity);
+      renderParticipant(participant);
+      renderScreenShare(room);
     })
-    .on(RoomEvent.TrackUnsubscribed, (_, pub) => {
+    .on(RoomEvent.TrackUnsubscribed, (_, pub, participant) => {
       appendLog('unsubscribed from track', pub.trackSid);
+      renderParticipant(participant);
+      renderScreenShare(room);
     })
     .on(RoomEvent.SignalConnected, async () => {
       const signalConnectionTime = Date.now() - startTime;
@@ -236,6 +513,7 @@ const connectToRoom = async (
   participantConnected(room.localParticipant);
   // 禁用视频
   await currentRoom.localParticipant.setCameraEnabled(false);
+  currentRoom?.startAudio();
 
   return room;
 };
@@ -258,12 +536,12 @@ function renderBitrate() {
         totalBitrate += t.track.currentBitrate;
       }
 
-      // if (t.source === Track.Source.Camera) {
-      //   if (t.videoTrack instanceof RemoteVideoTrack) {
-      //     // eslint-disable-next-line no-console
-      //     console.log(t.videoTrack.getDecoderImplementation() ?? '');
-      //   }
-      // }
+      if (t.source === Track.Source.Camera) {
+        if (t.videoTrack instanceof RemoteVideoTrack) {
+          // eslint-disable-next-line no-console
+          console.log(t.videoTrack.getDecoderImplementation() ?? '');
+        }
+      }
     }
     let displayText = '';
     if (totalBitrate > 0) {
@@ -284,9 +562,9 @@ const connectWithFormInput = async (
   const forceTURN = false; // 强制转向
   const adaptiveStream = true; // 自适应流
   const shouldPublish = true; // 发布
-  // const preferredCodec = '' as VideoCodec;
-  // const scalabilityMode = '';
-  // const cryptoKey = '';
+  const preferredCodec = '' as VideoCodec;
+  const scalabilityMode = '';
+  const cryptoKey = '';
   const autoSubscribe = true; // 自动订阅
   const e2eeEnabled = false;
   const audioOutputId = ''; // E2EE密钥
@@ -300,31 +578,31 @@ const connectWithFormInput = async (
     },
     publishDefaults: {
       simulcast,
-      // videoSimulcastLayers: [VideoPresets.h90, VideoPresets.h216],
-      // videoCodec: preferredCodec || 'vp8',
+      videoSimulcastLayers: [VideoPresets.h90, VideoPresets.h216],
+      videoCodec: preferredCodec || 'vp8',
       dtx: true,
       red: true,
       forceStereo: false,
       screenShareEncoding: ScreenSharePresets.h1080fps30.encoding,
       scalabilityMode: 'L3T3_KEY',
     },
-    // videoCaptureDefaults: {
-    //   resolution: VideoPresets.h720.resolution,
-    // },
+    videoCaptureDefaults: {
+      resolution: VideoPresets.h720.resolution,
+    },
     e2ee: e2eeEnabled
       ? { keyProvider: state.e2eeKeyProvider, worker: new E2EEWorker() }
       : undefined,
   };
-  // if (
-  //   roomOpts.publishDefaults?.videoCodec === 'av1' ||
-  //   roomOpts.publishDefaults?.videoCodec === 'vp9'
-  // ) {
-  //   roomOpts.publishDefaults.backupCodec = true;
-  //   if (scalabilityMode !== '') {
-  //     roomOpts.publishDefaults.scalabilityMode =
-  //       scalabilityMode as ScalabilityMode;
-  //   }
-  // }
+  if (
+    roomOpts.publishDefaults?.videoCodec === 'av1' ||
+    roomOpts.publishDefaults?.videoCodec === 'vp9'
+  ) {
+    roomOpts.publishDefaults.backupCodec = true;
+    if (scalabilityMode !== '') {
+      roomOpts.publishDefaults.scalabilityMode =
+        scalabilityMode as ScalabilityMode;
+    }
+  }
 
   const connectOpts: RoomConnectOptions = {
     autoSubscribe,
@@ -336,6 +614,7 @@ const connectWithFormInput = async (
   }
   await connectToRoom(url, token, roomOpts, connectOpts, shouldPublish);
   disabled.value = 'open';
+  timer = setTimeout(handleDevicesChanged, 100);
 
   state.bitrateInterval = setInterval(renderBitrate, 1000);
 };
@@ -350,9 +629,16 @@ const disconnectRoom = (disabled: Ref<string>) => {
   disabled.value = 'close';
 };
 
+async function acquireDeviceList() {
+  handleDevicesChanged();
+}
+
+acquireDeviceList();
+
 export const IntercomFn = async (disabled: Ref<string>, guid: string) => {
   if (disabled.value === 'open') {
     disconnectRoom(disabled);
+    clearInterval(timer);
     return;
   }
   disabled.value = 'loading';

src/views/preview-list/children/preview-info.vue

@@ -345,6 +345,7 @@
       :guid="data.id"
       :is-show-cl="false"
     />
+    <div id="participants-area" style="display: none"></div>
   </div>
 </template>