From 100b934b3e581b4f4cf5c8d1791b38b69e1ac863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez=20D=C3=ADez?= Date: Sun, 10 Apr 2022 11:41:36 +0200 Subject: [PATCH] [Voice] Allow hli list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miguel Álvarez Díez --- .../core/model/script/actions/Voice.java | 12 +-- .../org/openhab/core/voice/VoiceManager.java | 30 +++--- .../core/voice/internal/DialogProcessor.java | 28 +++--- .../VoiceConsoleCommandExtension.java | 13 ++- .../core/voice/internal/VoiceManagerImpl.java | 97 ++++++++++++------- 5 files changed, 111 insertions(+), 69 deletions(-) diff --git a/bundles/org.openhab.core.model.script/src/org/openhab/core/model/script/actions/Voice.java b/bundles/org.openhab.core.model.script/src/org/openhab/core/model/script/actions/Voice.java index 40961843fa8..922cb2769ee 100644 --- a/bundles/org.openhab.core.model.script/src/org/openhab/core/model/script/actions/Voice.java +++ b/bundles/org.openhab.core.model.script/src/org/openhab/core/model/script/actions/Voice.java @@ -157,7 +157,7 @@ public static String interpret(@ParamDoc(name = "text") Object text) { * In case of interpretation error, the error message is played using the default audio sink. * * @param text The text to interpret - * @param interpreter The Human Language Interpreter to be used + * @param interpreter Comma separate list of human language text interpreters to use or null to use the default services */ @ActionDoc(text = "interprets a given text by a given human language interpreter", returns = "human language response") public static String interpret(@ParamDoc(name = "text") Object text, @@ -180,7 +180,7 @@ public static String interpret(@ParamDoc(name = "text") Object text, * If sink parameter is null, the error message is simply not played. * * @param text The text to interpret - * @param interpreter The Human Language Interpreter to be used + * @param interpreter Comma separate list of human language text interpreters to use or null to use the default services * @param sink The name of audio sink to be used to play the error message */ @ActionDoc(text = "interprets a given text by a given human language interpreter", returns = "human language response") @@ -218,10 +218,10 @@ public static void startDialog(@ParamDoc(name = "source") @Nullable String sourc * @param ks the keyword spotting service to use or null to use the default service * @param stt the speech-to-text service to use or null to use the default service * @param tts the text-to-speech service to use or null to use the default service - * @param interpreter the human language text interpreter to use or null to use the default service + * @param interpreter comma separate list of human language text interpreters to use or null to use the default services * @param source the name of audio source to use or null to use the default source * @param sink the name of audio sink to use or null to use the default sink - * @param Locale the locale to use or null to use the default locale + * @param locale the locale to use or null to use the default locale * @param keyword the keyword to use during keyword spotting or null to use the default keyword * @param listeningItem the item to switch ON while listening to a question */ @@ -339,10 +339,10 @@ public static void listenAndAnswer(@ParamDoc(name = "source") @Nullable String s * * @param stt the speech-to-text service to use or null to use the default service * @param tts the text-to-speech service to use or null to use the default service - * @param interpreter the human language text interpreter to use or null to use the default service + * @param interpreter comma separate list of human language text interpreters to use or null to use the default services * @param source the name of audio source to use or null to use the default source * @param sink the name of audio sink to use or null to use the default sink - * @param Locale the locale to use or null to use the default locale + * @param locale the locale to use or null to use the default locale * @param listeningItem the item to switch ON while listening to a question */ @ActionDoc(text = "executes a simple dialog sequence without keyword spotting for a given audio source") diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/VoiceManager.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/VoiceManager.java index ae56765f3f1..8c033ed086d 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/VoiceManager.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/VoiceManager.java @@ -106,7 +106,7 @@ public interface VoiceManager { * Interprets the passed string using a particular HLI service and the default locale. * * @param text The text to interpret - * @param hliId The id of the HLI service to use or null + * @param hliId Comma separate list of HLI service ids to use or null * @throws InterpretationException * @return a human language response */ @@ -144,17 +144,17 @@ public interface VoiceManager { * @param ks the keyword spotting service to use or null to use the default service * @param stt the speech-to-text service to use or null to use the default service * @param tts the text-to-speech service to use or null to use the default service - * @param hli the human language text interpreter to use or null to use the default service + * @param hli collection of human language text interpreters to use or null to use the default services * @param source the audio source to use or null to use the default source * @param sink the audio sink to use or null to use the default sink - * @param Locale the locale to use or null to use the default locale + * @param locale the locale to use or null to use the default locale * @param keyword the keyword to use during keyword spotting or null to use the default keyword * @param listeningItem the item to switch ON while listening to a question * @throws IllegalStateException if required services are not all available or the provided locale is not supported * by all these services or a dialog is already started for this audio source */ void startDialog(@Nullable KSService ks, @Nullable STTService stt, @Nullable TTSService tts, - @Nullable HumanLanguageInterpreter hli, @Nullable AudioSource source, @Nullable AudioSink sink, + @Nullable Collection hli, @Nullable AudioSource source, @Nullable AudioSink sink, @Nullable Locale locale, @Nullable String keyword, @Nullable String listeningItem) throws IllegalStateException; @@ -187,7 +187,7 @@ void startDialog(@Nullable KSService ks, @Nullable STTService stt, @Nullable TTS * * @param stt the speech-to-text service to use or null to use the default service * @param tts the text-to-speech service to use or null to use the default service - * @param hli the human language text interpreter to use or null to use the default service + * @param hli collection of human language text interpreters to use or null to use the default services * @param source the audio source to use or null to use the default source * @param sink the audio sink to use or null to use the default sink * @param locale the locale to use or null to use the default locale @@ -195,9 +195,9 @@ void startDialog(@Nullable KSService ks, @Nullable STTService stt, @Nullable TTS * @throws IllegalStateException if required services are not all available or the provided locale is not supported * by all these services or a dialog is already started for this audio source */ - void listenAndAnswer(@Nullable STTService stt, @Nullable TTSService tts, @Nullable HumanLanguageInterpreter hli, - @Nullable AudioSource source, @Nullable AudioSink sink, @Nullable Locale locale, - @Nullable String listeningItem) throws IllegalStateException; + void listenAndAnswer(@Nullable STTService stt, @Nullable TTSService tts, + @Nullable Collection hli, @Nullable AudioSource source, @Nullable AudioSink sink, + @Nullable Locale locale, @Nullable String listeningItem) throws IllegalStateException; /** * Retrieves a TTS service. @@ -281,15 +281,17 @@ void listenAndAnswer(@Nullable STTService stt, @Nullable TTSService tts, @Nullab Collection getKSs(); /** - * Retrieves a HumanLanguageInterpreter. - * If a default name is configured and the service available, this is returned. Otherwise, the first available + * Retrieves a HumanLanguageInterpreter collection. + * If a default names are configured and the services are available, those are returned. Otherwise, the first + * available * service is returned. * - * @return a HumanLanguageInterpreter or null, if no service is available or if a default is configured, but no - * according service is found + * @return a Collection or null, if no services are available or if some defaults are + * configured, but no + * according services are found */ @Nullable - HumanLanguageInterpreter getHLI(); + Collection getHLI(); /** * Retrieves a HumanLanguageInterpreter with the given id. @@ -298,7 +300,7 @@ void listenAndAnswer(@Nullable STTService stt, @Nullable TTSService tts, @Nullab * @return a HumanLanguageInterpreter or null, if no interpreter with this id exists */ @Nullable - HumanLanguageInterpreter getHLI(String id); + Collection getHLI(String id); /** * Retrieves all HumanLanguageInterpreters. diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/DialogProcessor.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/DialogProcessor.java index d89024e5d60..041ea8060c2 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/DialogProcessor.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/DialogProcessor.java @@ -13,6 +13,7 @@ package org.openhab.core.voice.internal; import java.io.IOException; +import java.util.Collection; import java.util.HashSet; import java.util.Locale; @@ -74,7 +75,7 @@ public class DialogProcessor implements KSListener, STTListener { private final @Nullable KSService ks; private final STTService stt; private final TTSService tts; - private final HumanLanguageInterpreter hli; + private final Collection hli; private final AudioSource source; private final AudioSink sink; private final Locale locale; @@ -104,7 +105,7 @@ public class DialogProcessor implements KSListener, STTListener { private @Nullable AudioStream streamKS; private @Nullable AudioStream streamSTT; - public DialogProcessor(KSService ks, STTService stt, TTSService tts, HumanLanguageInterpreter hli, + public DialogProcessor(KSService ks, STTService stt, TTSService tts, Collection hli, AudioSource source, AudioSink sink, Locale locale, String keyword, @Nullable String listeningItem, EventPublisher eventPublisher, TranslationProvider i18nProvider, Bundle bundle) { this.locale = locale; @@ -124,7 +125,7 @@ public DialogProcessor(KSService ks, STTService stt, TTSService tts, HumanLangua this.ttsFormat = VoiceManagerImpl.getBestMatch(tts.getSupportedFormats(), sink.getSupportedFormats()); } - public DialogProcessor(STTService stt, TTSService tts, HumanLanguageInterpreter hli, AudioSource source, + public DialogProcessor(STTService stt, TTSService tts, Collection hli, AudioSource source, AudioSink sink, Locale locale, @Nullable String listeningItem, EventPublisher eventPublisher, TranslationProvider i18nProvider, Bundle bundle) { this.locale = locale; @@ -282,15 +283,20 @@ public synchronized void sttEventReceived(STTEvent sttEvent) { SpeechRecognitionEvent sre = (SpeechRecognitionEvent) sttEvent; String question = sre.getTranscript(); logger.debug("Text recognized: {}", question); - try { - toggleProcessing(false); - String answer = hli.interpret(locale, question); - logger.debug("Interpretation result: {}", answer); - say(answer); - } catch (InterpretationException e) { - logger.debug("Interpretation exception: {}", e.getMessage()); - say(e.getMessage()); + String answer = null; + String error = null; + for (var interpreter : hli) { + try { + answer = interpreter.interpret(locale, question); + logger.debug("Interpretation result: {}", answer); + break; + } catch (InterpretationException e) { + logger.debug("Interpretation exception: {}", e.getMessage()); + error = e.getMessage(); + } } + toggleProcessing(false); + say(answer != null ? answer : error); abortSTT(); } } else if (sttEvent instanceof RecognitionStartEvent) { diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceConsoleCommandExtension.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceConsoleCommandExtension.java index bf8affce14a..caf3eafe689 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceConsoleCommandExtension.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceConsoleCommandExtension.java @@ -135,7 +135,8 @@ public void execute(String[] args, Console console) { try { AudioSource source = args.length < 2 ? null : audioManager.getSource(args[1]); AudioSink sink = args.length < 3 ? null : audioManager.getSink(args[2]); - HumanLanguageInterpreter hli = args.length < 4 ? null : voiceManager.getHLI(args[3]); + Collection hli = args.length < 4 ? null + : voiceManager.getHLI(args[3]); TTSService tts = args.length < 5 ? null : voiceManager.getTTS(args[4]); STTService stt = args.length < 6 ? null : voiceManager.getSTT(args[5]); KSService ks = args.length < 7 ? null : voiceManager.getKS(args[6]); @@ -158,7 +159,8 @@ public void execute(String[] args, Console console) { try { AudioSource source = args.length < 2 ? null : audioManager.getSource(args[1]); AudioSink sink = args.length < 3 ? null : audioManager.getSink(args[2]); - HumanLanguageInterpreter hli = args.length < 4 ? null : voiceManager.getHLI(args[3]); + Collection hli = args.length < 4 ? null + : voiceManager.getHLI(args[3]); TTSService tts = args.length < 5 ? null : voiceManager.getTTS(args[4]); STTService stt = args.length < 6 ? null : voiceManager.getSTT(args[5]); voiceManager.listenAndAnswer(stt, tts, hli, source, sink, null, null); @@ -243,11 +245,12 @@ private void say(String[] args, Console console) { private void listInterpreters(Console console) { Collection interpreters = voiceManager.getHLIs(); if (!interpreters.isEmpty()) { - HumanLanguageInterpreter defaultHLI = voiceManager.getHLI(); + var defaultHLIs = voiceManager.getHLI(); Locale locale = localeProvider.getLocale(); interpreters.stream().sorted(comparing(s -> s.getLabel(locale))).forEach(hli -> { - console.println(String.format("%s %s (%s)", hli.equals(defaultHLI) ? "*" : " ", hli.getLabel(locale), - hli.getId())); + console.println(String.format("%s %s (%s)", + (defaultHLIs != null && defaultHLIs.stream().anyMatch(hli::equals)) ? "*" : " ", + hli.getLabel(locale), hli.getId())); }); } else { console.println("No interpreters found."); diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceManagerImpl.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceManagerImpl.java index 178c925d48a..3bc6fbbbc78 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceManagerImpl.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceManagerImpl.java @@ -288,20 +288,36 @@ public String interpret(String text) throws InterpretationException { } @Override - public String interpret(String text, @Nullable String hliId) throws InterpretationException { - HumanLanguageInterpreter interpreter; - if (hliId == null) { - interpreter = getHLI(); - if (interpreter == null) { + public String interpret(String text, @Nullable String hliIdList) throws InterpretationException { + Collection interpreters; + if (hliIdList == null) { + interpreters = getHLI(); + if (interpreters == null) { throw new InterpretationException("No human language interpreter available!"); } } else { - interpreter = getHLI(hliId); - if (interpreter == null) { - throw new InterpretationException("No human language interpreter can be found for " + hliId); + interpreters = getHLI(hliIdList); + if (interpreters == null) { + throw new InterpretationException("No human language interpreter can be found for " + hliIdList); } } - return interpreter.interpret(localeProvider.getLocale(), text); + String answer = null; + InterpretationException error = null; + Locale locale = localeProvider.getLocale(); + for (var interpreter : interpreters) { + try { + answer = interpreter.interpret(locale, text); + logger.debug("Interpretation result: {}", answer); + break; + } catch (InterpretationException e) { + logger.debug("Interpretation exception: {}", e.getMessage()); + error = e; + } + } + if (answer == null && error != null) { + throw error; + } + return answer; } private @Nullable Voice getVoice(String id) { @@ -485,14 +501,14 @@ public void startDialog() throws IllegalStateException { @Override public void startDialog(@Nullable KSService ks, @Nullable STTService stt, @Nullable TTSService tts, - @Nullable HumanLanguageInterpreter hli, @Nullable AudioSource source, @Nullable AudioSink sink, + @Nullable Collection hli, @Nullable AudioSource source, @Nullable AudioSink sink, @Nullable Locale locale, @Nullable String keyword, @Nullable String listeningItem) throws IllegalStateException { // use defaults, if null KSService ksService = (ks == null) ? getKS() : ks; STTService sttService = (stt == null) ? getSTT() : stt; TTSService ttsService = (tts == null) ? getTTS() : tts; - HumanLanguageInterpreter interpreter = (hli == null) ? getHLI() : hli; + Collection interpreters = (hli == null) ? getHLI() : hli; AudioSource audioSource = (source == null) ? audioManager.getSource() : source; AudioSink audioSink = (sink == null) ? audioManager.getSink() : sink; Locale loc = (locale == null) ? localeProvider.getLocale() : locale; @@ -500,19 +516,19 @@ public void startDialog(@Nullable KSService ks, @Nullable STTService stt, @Nulla String item = (listeningItem == null) ? this.listeningItem : listeningItem; Bundle b = bundle; - if (ksService == null || sttService == null || ttsService == null || interpreter == null || audioSource == null + if (ksService == null || sttService == null || ttsService == null || interpreters == null || audioSource == null || audioSink == null || b == null) { throw new IllegalStateException("Cannot start dialog as services are missing."); } else if (!checkLocales(ksService.getSupportedLocales(), loc) - || !checkLocales(sttService.getSupportedLocales(), loc) - || !checkLocales(interpreter.getSupportedLocales(), loc)) { + || !checkLocales(sttService.getSupportedLocales(), loc) || !interpreters.stream() + .allMatch(interpreter -> checkLocales(interpreter.getSupportedLocales(), loc))) { throw new IllegalStateException("Cannot start dialog as provided locale is not supported by all services."); } else { DialogProcessor processor = dialogProcessors.get(audioSource.getId()); if (processor == null) { logger.debug("Starting a new dialog for source {} ({})", audioSource.getLabel(null), audioSource.getId()); - processor = new DialogProcessor(ksService, sttService, ttsService, interpreter, audioSource, audioSink, + processor = new DialogProcessor(ksService, sttService, ttsService, interpreters, audioSource, audioSink, loc, kw, item, this.eventPublisher, this.i18nProvider, b); dialogProcessors.put(audioSource.getId(), processor); processor.start(); @@ -556,23 +572,23 @@ public void listenAndAnswer() throws IllegalStateException { @Override public void listenAndAnswer(@Nullable STTService stt, @Nullable TTSService tts, - @Nullable HumanLanguageInterpreter hli, @Nullable AudioSource source, @Nullable AudioSink sink, + @Nullable Collection hli, @Nullable AudioSource source, @Nullable AudioSink sink, @Nullable Locale locale, @Nullable String listeningItem) throws IllegalStateException { // use defaults, if null STTService sttService = (stt == null) ? getSTT() : stt; TTSService ttsService = (tts == null) ? getTTS() : tts; - HumanLanguageInterpreter interpreter = (hli == null) ? getHLI() : hli; + Collection interpreters = (hli == null) ? getHLI() : hli; AudioSource audioSource = (source == null) ? audioManager.getSource() : source; AudioSink audioSink = (sink == null) ? audioManager.getSink() : sink; Locale loc = (locale == null) ? localeProvider.getLocale() : locale; String item = (listeningItem == null) ? this.listeningItem : listeningItem; Bundle b = bundle; - if (sttService == null || ttsService == null || interpreter == null || audioSource == null || audioSink == null + if (sttService == null || ttsService == null || interpreters == null || audioSource == null || audioSink == null || b == null) { throw new IllegalStateException("Cannot execute a simple dialog as services are missing."); - } else if (!checkLocales(sttService.getSupportedLocales(), loc) - || !checkLocales(interpreter.getSupportedLocales(), loc)) { + } else if (!checkLocales(sttService.getSupportedLocales(), loc) || !interpreters.stream() + .allMatch(interpreter -> checkLocales(interpreter.getSupportedLocales(), loc))) { throw new IllegalStateException( "Cannot execute a simple dialog as provided locale is not supported by all services."); } else { @@ -580,7 +596,7 @@ public void listenAndAnswer(@Nullable STTService stt, @Nullable TTSService tts, if (processor == null) { logger.debug("Executing a simple dialog for source {} ({})", audioSource.getLabel(null), audioSource.getId()); - processor = new DialogProcessor(sttService, ttsService, interpreter, audioSource, audioSink, loc, item, + processor = new DialogProcessor(sttService, ttsService, interpreters, audioSource, audioSink, loc, item, this.eventPublisher, this.i18nProvider, b); processor.start(); } else { @@ -721,24 +737,39 @@ public Collection getKSs() { } @Override - public @Nullable HumanLanguageInterpreter getHLI() { - HumanLanguageInterpreter hli = null; + public @Nullable Collection getHLI() { + var defaultHLI = this.defaultHLI; if (defaultHLI != null) { - hli = humanLanguageInterpreters.get(defaultHLI); - if (hli == null) { - logger.warn("Default HumanLanguageInterpreter '{}' not available!", defaultHLI); + var interpreters = this.getHLI(defaultHLI); + if (interpreters != null && !interpreters.isEmpty()) { + return interpreters; + } + } + if (!humanLanguageInterpreters.isEmpty()) { + var hli = humanLanguageInterpreters.values().iterator().next(); + if (hli != null) { + return List.of(hli); } - } else if (!humanLanguageInterpreters.isEmpty()) { - hli = humanLanguageInterpreters.values().iterator().next(); - } else { - logger.debug("No HumanLanguageInterpreter available!"); } - return hli; + logger.debug("No HumanLanguageInterpreter available!"); + return null; } @Override - public @Nullable HumanLanguageInterpreter getHLI(String id) { - return humanLanguageInterpreters.get(id); + public @Nullable Collection getHLI(String idList) { + List interpreters = new ArrayList<>(); + for (var id : idList.split(",")) { + var hli = humanLanguageInterpreters.get(id); + if (hli == null) { + logger.warn("Default HumanLanguageInterpreter '{}' not available!", id); + } else { + interpreters.add(hli); + } + } + if (interpreters.isEmpty()) { + return null; + } + return interpreters; } @Override